From 2c1c9353b77b471aef9815c16eaa5cf35a415645 Mon Sep 17 00:00:00 2001
From: rasbt <mail@sebastianraschka.com>
Date: Fri, 15 Nov 2024 09:33:35 +0900
Subject: [PATCH] Deployed 41a5453b with MkDocs version: 1.5.3

---
 404.html                                      |  33 +-
 CHANGELOG/index.html                          | 186 ++++----
 CONTRIBUTING/index.html                       |  33 +-
 Code-of-Conduct/index.html                    |  33 +-
 USER-GUIDE-INDEX/index.html                   |  33 +-
 USER_GUIDE_INDEX/index.html                   |  33 +-
 .../mlxtend.classifier/Adaline/index.html     |  33 +-
 .../EnsembleVoteClassifier/index.html         |  33 +-
 .../LogisticRegression/index.html             |  33 +-
 .../MultiLayerPerceptron/index.html           |  33 +-
 .../OneRClassifier/index.html                 |  33 +-
 .../mlxtend.classifier/Perceptron/index.html  |  33 +-
 .../SoftmaxRegression/index.html              |  33 +-
 .../StackingCVClassifier/index.html           |  33 +-
 .../StackingClassifier/index.html             |  33 +-
 api_modules/mlxtend.cluster/Kmeans/index.html |  33 +-
 .../mlxtend.data/autompg_data/index.html      |  33 +-
 .../boston_housing_data/index.html            |  33 +-
 api_modules/mlxtend.data/iris_data/index.html |  33 +-
 .../mlxtend.data/loadlocal_mnist/index.html   |  33 +-
 .../make_multiplexer_dataset/index.html       |  33 +-
 .../mlxtend.data/mnist_data/index.html        |  33 +-
 .../mlxtend.data/three_blobs_data/index.html  |  33 +-
 api_modules/mlxtend.data/wine_data/index.html |  33 +-
 .../BootstrapOutOfBag/index.html              |  33 +-
 .../GroupTimeSeriesSplit/index.html           |  33 +-
 .../PredefinedHoldoutSplit/index.html         |  33 +-
 .../RandomHoldoutSplit/index.html             |  33 +-
 .../accuracy_score/index.html                 |  33 +-
 .../bias_variance_decomp/index.html           |  33 +-
 .../mlxtend.evaluate/bootstrap/index.html     |  33 +-
 .../bootstrap_point632_score/index.html       |  33 +-
 .../mlxtend.evaluate/cochrans_q/index.html    |  33 +-
 .../combined_ftest_5x2cv/index.html           |  33 +-
 .../confusion_matrix/index.html               |  33 +-
 .../create_counterfactual/index.html          |  33 +-
 .../feature_importance_permutation/index.html |  33 +-
 api_modules/mlxtend.evaluate/ftest/index.html |  33 +-
 .../mlxtend.evaluate/lift_score/index.html    |  33 +-
 .../mlxtend.evaluate/mcnemar/index.html       |  33 +-
 .../mlxtend.evaluate/mcnemar_table/index.html |  33 +-
 .../mcnemar_tables/index.html                 |  33 +-
 .../paired_ttest_5x2cv/index.html             |  33 +-
 .../paired_ttest_kfold_cv/index.html          |  33 +-
 .../paired_ttest_resampled/index.html         |  33 +-
 .../permutation_test/index.html               |  33 +-
 .../proportion_difference/index.html          |  33 +-
 .../mlxtend.evaluate/scoring/index.html       |  33 +-
 .../LinearDiscriminantAnalysis/index.html     |  33 +-
 .../PrincipalComponentAnalysis/index.html     |  33 +-
 .../RBFKernelPCA/index.html                   |  33 +-
 .../ColumnSelector/index.html                 |  33 +-
 .../ExhaustiveFeatureSelector/index.html      |  33 +-
 .../SequentialFeatureSelector/index.html      |  33 +-
 .../find_filegroups/index.html                |  33 +-
 .../mlxtend.file_io/find_files/index.html     |  33 +-
 .../apriori/index.html                        |  33 +-
 .../association_rules/index.html              |  33 +-
 .../fpgrowth/index.html                       |  33 +-
 .../fpmax/index.html                          |  33 +-
 .../hmine/index.html                          |  33 +-
 .../mlxtend.image/EyepadAlign/index.html      |  33 +-
 .../extract_face_landmarks/index.html         |  33 +-
 api_modules/mlxtend.math/factorial/index.html |  33 +-
 .../mlxtend.math/num_combinations/index.html  |  33 +-
 .../mlxtend.math/num_permutations/index.html  |  33 +-
 .../vectorspace_dimensionality/index.html     |  33 +-
 .../vectorspace_orthonormalization/index.html |  33 +-
 .../category_scatter/index.html               |  33 +-
 .../checkerboard_plot/index.html              |  33 +-
 api_modules/mlxtend.plotting/ecdf/index.html  |  33 +-
 .../enrichment_plot/index.html                |  33 +-
 .../mlxtend.plotting/heatmap/index.html       |  33 +-
 .../plot_confusion_matrix/index.html          |  33 +-
 .../plot_decision_regions/index.html          |  33 +-
 .../plot_learning_curves/index.html           |  33 +-
 .../plot_linear_regression/index.html         |  33 +-
 .../plot_pca_correlation_graph/index.html     |  33 +-
 .../index.html                                |  33 +-
 .../remove_borders/index.html                 |  33 +-
 .../mlxtend.plotting/scatter_hist/index.html  |  33 +-
 .../scatterplotmatrix/index.html              |  33 +-
 .../stacked_barplot/index.html                |  33 +-
 .../CopyTransformer/index.html                |  33 +-
 .../DenseTransformer/index.html               |  33 +-
 .../MeanCenterer/index.html                   |  33 +-
 .../TransactionEncoder/index.html             |  33 +-
 .../minmax_scaling/index.html                 |  33 +-
 .../mlxtend.preprocessing/one_hot/index.html  |  33 +-
 .../shuffle_arrays_unison/index.html          |  33 +-
 .../standardize/index.html                    |  33 +-
 .../LinearRegression/index.html               |  33 +-
 .../StackingCVRegressor/index.html            |  33 +-
 .../StackingRegressor/index.html              |  33 +-
 .../mlxtend.text/generalize_names/index.html  |  33 +-
 .../generalize_names_duplcheck/index.html     |  33 +-
 .../tokenizer_emoticons/index.html            |  33 +-
 .../tokenizer_words_and_emoticons/index.html  |  33 +-
 api_modules/mlxtend.utils/Counter/index.html  |  33 +-
 .../mlxtend.utils/assert_raises/index.html    |  33 +-
 api_modules/mlxtend.utils/check_Xy/index.html |  33 +-
 .../format_kwarg_dictionaries/index.html      |  33 +-
 api_subpackages/mlxtend._base/index.html      |  33 +-
 api_subpackages/mlxtend.classifier/index.html |  33 +-
 api_subpackages/mlxtend.cluster/index.html    |  33 +-
 api_subpackages/mlxtend.data/index.html       |  33 +-
 api_subpackages/mlxtend.evaluate/index.html   |  33 +-
 api_subpackages/mlxtend.externals/index.html  |  33 +-
 .../mlxtend.feature_extraction/index.html     |  33 +-
 .../mlxtend.feature_selection/index.html      |  33 +-
 api_subpackages/mlxtend.file_io/index.html    |  33 +-
 .../mlxtend.frequent_patterns/index.html      |  33 +-
 api_subpackages/mlxtend.image/index.html      |  33 +-
 api_subpackages/mlxtend.math/index.html       |  33 +-
 api_subpackages/mlxtend.plotting/index.html   |  33 +-
 .../mlxtend.preprocessing/index.html          |  33 +-
 api_subpackages/mlxtend.regressor/index.html  |  33 +-
 api_subpackages/mlxtend.text/index.html       |  33 +-
 api_subpackages/mlxtend.utils/index.html      |  33 +-
 cite/index.html                               |  33 +-
 contributors/index.html                       |  33 +-
 discuss/index.html                            |  33 +-
 index.html                                    |  37 +-
 installation/index.html                       |  33 +-
 js/jquery-1.10.2.min.js                       |   6 -
 js/jquery-3.6.0.min.js                        |   2 +
 license/index.html                            |  33 +-
 search/search_index.json                      |   2 +-
 sitemap.xml                                   | 428 +++++++++---------
 sitemap.xml.gz                                | Bin 1969 -> 1969 bytes
 user_guide/classifier/Adaline/index.html      |  33 +-
 .../EnsembleVoteClassifier/index.html         |  33 +-
 .../classifier/LogisticRegression/index.html  |  33 +-
 .../MultiLayerPerceptron/index.html           |  33 +-
 .../classifier/OneRClassifier/index.html      |  33 +-
 user_guide/classifier/Perceptron/index.html   |  33 +-
 .../classifier/SoftmaxRegression/index.html   |  33 +-
 .../StackingCVClassifier/index.html           |  33 +-
 .../classifier/StackingClassifier/index.html  |  33 +-
 user_guide/cluster/Kmeans/index.html          |  33 +-
 user_guide/data/autompg_data/index.html       |  33 +-
 .../data/boston_housing_data/index.html       |  33 +-
 user_guide/data/iris_data/index.html          |  33 +-
 user_guide/data/loadlocal_mnist/index.html    |  33 +-
 .../data/make_multiplexer_dataset/index.html  |  33 +-
 user_guide/data/mnist_data/index.html         |  33 +-
 user_guide/data/three_blobs_data/index.html   |  33 +-
 user_guide/data/wine_data/index.html          |  33 +-
 .../evaluate/BootstrapOutOfBag/index.html     |  33 +-
 .../evaluate/GroupTimeSeriesSplit/index.html  |  33 +-
 .../PredefinedHoldoutSplit/index.html         |  33 +-
 .../evaluate/RandomHoldoutSplit/index.html    |  33 +-
 user_guide/evaluate/accuracy_score/index.html |  33 +-
 .../evaluate/bias_variance_decomp/index.html  |  33 +-
 user_guide/evaluate/bootstrap/index.html      |  33 +-
 .../bootstrap_point632_score/index.html       |  33 +-
 user_guide/evaluate/cochrans_q/index.html     |  33 +-
 .../evaluate/combined_ftest_5x2cv/index.html  |  35 +-
 .../evaluate/confusion_matrix/index.html      |  33 +-
 .../evaluate/create_counterfactual/index.html |  33 +-
 .../feature_importance_permutation/index.html |  33 +-
 user_guide/evaluate/ftest/index.html          |  33 +-
 user_guide/evaluate/lift_score/index.html     |  33 +-
 user_guide/evaluate/mcnemar/index.html        |  33 +-
 user_guide/evaluate/mcnemar_table/index.html  |  33 +-
 user_guide/evaluate/mcnemar_tables/index.html |  33 +-
 .../evaluate/paired_ttest_5x2cv/index.html    |  35 +-
 .../evaluate/paired_ttest_kfold_cv/index.html |  35 +-
 .../paired_ttest_resampled/index.html         |  35 +-
 .../evaluate/permutation_test/index.html      |  33 +-
 .../evaluate/proportion_difference/index.html |  33 +-
 user_guide/evaluate/scoring/index.html        |  33 +-
 .../LinearDiscriminantAnalysis/index.html     |  33 +-
 .../PrincipalComponentAnalysis/index.html     |  33 +-
 .../RBFKernelPCA/index.html                   |  33 +-
 .../ColumnSelector/index.html                 |  33 +-
 .../ExhaustiveFeatureSelector/index.html      |  33 +-
 .../SequentialFeatureSelector/index.html      |  33 +-
 user_guide/file_io/find_filegroups/index.html |  33 +-
 user_guide/file_io/find_files/index.html      |  33 +-
 .../frequent_patterns/apriori/index.html      |  33 +-
 .../association_rules/index.html              |  33 +-
 .../frequent_patterns/fpgrowth/index.html     |  33 +-
 user_guide/frequent_patterns/fpmax/index.html |  33 +-
 user_guide/frequent_patterns/hmine/index.html |  33 +-
 .../image/extract_face_landmarks/index.html   |  33 +-
 user_guide/image/eyepad_align/index.html      |  33 +-
 user_guide/math/num_combinations/index.html   |  35 +-
 user_guide/math/num_permutations/index.html   |  35 +-
 .../vectorspace_dimensionality/index.html     |  33 +-
 .../vectorspace_orthonormalization/index.html |  33 +-
 .../plotting/category_scatter/index.html      |  33 +-
 .../plotting/checkerboard_plot/index.html     |  33 +-
 user_guide/plotting/ecdf/index.html           |  33 +-
 .../plotting/enrichment_plot/index.html       |  33 +-
 user_guide/plotting/heatmap/index.html        |  33 +-
 .../plotting/plot_confusion_matrix/index.html |  33 +-
 .../plotting/plot_decision_regions/index.html |  33 +-
 .../plotting/plot_learning_curves/index.html  |  33 +-
 .../plot_linear_regression/index.html         |  33 +-
 .../plot_pca_correlation_graph/index.html     |  33 +-
 .../index.html                                |  33 +-
 user_guide/plotting/scatter_hist/index.html   |  33 +-
 .../plotting/scatterplotmatrix/index.html     |  33 +-
 .../plotting/stacked_barplot/index.html       |  33 +-
 .../preprocessing/CopyTransformer/index.html  |  33 +-
 .../preprocessing/DenseTransformer/index.html |  33 +-
 .../preprocessing/MeanCenterer/index.html     |  33 +-
 .../TransactionEncoder/index.html             |  33 +-
 .../preprocessing/minmax_scaling/index.html   |  33 +-
 .../preprocessing/one-hot_encoding/index.html |  33 +-
 .../shuffle_arrays_unison/index.html          |  33 +-
 .../preprocessing/standardize/index.html      |  33 +-
 .../regressor/LinearRegression/index.html     |  33 +-
 .../regressor/StackingCVRegressor/index.html  |  37 +-
 .../regressor/StackingRegressor/index.html    |  37 +-
 user_guide/text/generalize_names/index.html   |  33 +-
 .../generalize_names_duplcheck/index.html     |  33 +-
 user_guide/text/tokenizer/index.html          |  33 +-
 user_guide/utils/Counter/index.html           |  33 +-
 220 files changed, 3756 insertions(+), 3954 deletions(-)
 delete mode 100644 js/jquery-1.10.2.min.js
 create mode 100644 js/jquery-3.6.0.min.js

diff --git a/404.html b/404.html
index ee44d90c9..ec3f4c34d 100644
--- a/404.html
+++ b/404.html
@@ -12,25 +12,22 @@
         <link href="/mlxtend/css/bootstrap.min.css" rel="stylesheet">
         <link href="/mlxtend/css/font-awesome.min.css" rel="stylesheet">
         <link href="/mlxtend/css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="/mlxtend/cinder/css/base.css" rel="stylesheet">
         <link href="/mlxtend/cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="/mlxtend/cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="/mlxtend/cinder/css/cinder.css" rel="stylesheet">
         <link href="/mlxtend/cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="/mlxtend/cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="/mlxtend/js/jquery-1.10.2.min.js" defer></script>
-        <script src="/mlxtend/js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="/mlxtend/user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="/mlxtend/user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="/mlxtend/user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="/mlxtend/user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="/mlxtend/user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="/mlxtend/user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="/mlxtend/user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="/mlxtend/user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="/mlxtend/user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="/mlxtend/user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="/mlxtend/user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="/mlxtend/user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -616,14 +613,16 @@ <h1 id="404-page-not-found" style="text-align: center">404</h1>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="/mlxtend/js/jquery-3.6.0.min.js"></script>
+        <script src="/mlxtend/js/bootstrap.min.js"></script>
         <script>
             var base_url = "/mlxtend/",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="/mlxtend/js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="/mlxtend/mathjaxhelper.js" defer></script>
-        <script src="/mlxtend/search/main.js" defer></script>
+        <script src="/mlxtend/js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="/mlxtend/mathjaxhelper.js"></script>
+        <script src="/mlxtend/search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/CHANGELOG/index.html b/CHANGELOG/index.html
index 3de279b29..0c38e36fe 100644
--- a/CHANGELOG/index.html
+++ b/CHANGELOG/index.html
@@ -12,25 +12,22 @@
         <link href="../css/bootstrap.min.css" rel="stylesheet">
         <link href="../css/font-awesome.min.css" rel="stylesheet">
         <link href="../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../cinder/css/base.css" rel="stylesheet">
         <link href="../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../cinder/css/cinder.css" rel="stylesheet">
         <link href="../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -634,17 +631,38 @@ <h1 id="release-notes">Release Notes</h1>
 <p>The CHANGELOG for the current development version is available at
 <a href="https://github.com/rasbt/mlxtend/blob/master/docs/sources/CHANGELOG.md">https://github.com/rasbt/mlxtend/blob/master/docs/sources/CHANGELOG.md</a>.</p>
 <hr />
-<h3 id="version-0232-5-nov-2024">Version 0.23.2  (5 Nov 2024)</h3>
+<h3 id="version-0233-15-nov-2024">Version 0.23.3  (15 Nov 2024)</h3>
 <h5 id="downloads">Downloads</h5>
 <ul>
 <li>
+<p><a href="https://github.com/rasbt/mlxtend/archive/v0.23.3.zip">Source code (zip)</a></p>
+</li>
+<li>
+<p><a href="https://github.com/rasbt/mlxtend/archive/v0.23.3.tar.gz">Source code (tar.gz)</a></p>
+</li>
+</ul>
+<h5 id="new-features-and-enhancements">New Features and Enhancements</h5>
+<p>Files updated:
+  - <a href="https://github.com/rasbt/mlxtend/blob/master/mlxtend/evaluate/time_series.py"><code>mlxtend.evaluate.time_series.plot_splits</code></a>
+    - Improved <code>plot_splits</code> for better visualization of time series splits</p>
+<h5 id="changes">Changes</h5>
+<ul>
+<li><a href="https://github.com/rasbt/mlxtend/blob/master/mlxtend/feature_selection/exhaustive_feature_selector.py"><code>mlxtend/feature_selection/exhaustive_feature_selector.py</code></a><ul>
+<li>np.inf update to support for NumPy 2.0</li>
+</ul>
+</li>
+</ul>
+<h3 id="version-0232-5-nov-2024">Version 0.23.2  (5 Nov 2024)</h3>
+<h5 id="downloads_1">Downloads</h5>
+<ul>
+<li>
 <p><a href="https://github.com/rasbt/mlxtend/archive/v0.23.2.zip">Source code (zip)</a></p>
 </li>
 <li>
 <p><a href="https://github.com/rasbt/mlxtend/archive/v0.23.2.tar.gz">Source code (tar.gz)</a></p>
 </li>
 </ul>
-<h5 id="new-features-and-enhancements">New Features and Enhancements</h5>
+<h5 id="new-features-and-enhancements_1">New Features and Enhancements</h5>
 <ul>
 <li>Implement the FP-Growth and FP-Max algorithms with the possibility of missing values in the input dataset. Added a new metric Representativity for the association rules generated (<a href="https://github.com/rasbt/mlxtend/issues/1004">#1004</a> via <a href="https://github.com/zazass8">zazass8</a>).
   Files updated:</li>
@@ -665,7 +683,7 @@ <h5 id="new-features-and-enhancements">New Features and Enhancements</h5>
 </li>
 <li>Integrated scikit-learn's <code>set_output</code> method into <code>TransactionEncoder</code> (<a href="https://github.com/rasbt/mlxtend/issues/1087">#1087</a> via <a href="https://github.com/it176131">it176131</a>)</li>
 </ul>
-<h5 id="changes">Changes</h5>
+<h5 id="changes_1">Changes</h5>
 <ul>
 <li>[<code>mlxtend.frequent_patterns.fpcommon</code>] Added the null_values parameter in valid_input_check signature to check in case the input also includes null values. Changes the returns statements and function signatures for setup_fptree and generated_itemsets respectively to return the disabled array created and to include it as a parameter. Added code in [<code>mlxtend.frequent_patterns.fpcommon</code>] and <a href="https://rasbt.github.io/mlxtend/user_guide/frequent_patterns/association_rules/"><code>mlxtend.frequent_patterns.association_rules</code></a> to implement the algorithms in case null values exist when null_values is True.</li>
 <li>
@@ -677,7 +695,7 @@ <h5 id="changes">Changes</h5>
 <li>Use Scipy's instead of NumPy's decompositions in PCA for improved accuracy in edge cases (<a href="https://github.com/rasbt/mlxtend/issues/1080">#1080</a> via [<a href="https://github.com/rasbt/mlxtend/issues?q=is%3Apr+is%3Aopen+author%3Afkdosilovic">fkdosilovic</a>])</li>
 </ul>
 <h3 id="version-0231-5-jan-2024">Version 0.23.1 (5 Jan 2024)</h3>
-<h5 id="downloads_1">Downloads</h5>
+<h5 id="downloads_2">Downloads</h5>
 <ul>
 <li>
 <p><a href="https://github.com/rasbt/mlxtend/archive/v0.23.1.zip">Source code (zip)</a></p>
@@ -686,12 +704,12 @@ <h5 id="downloads_1">Downloads</h5>
 <p><a href="https://github.com/rasbt/mlxtend/archive/v0.23.1.tar.gz">Source code (tar.gz)</a></p>
 </li>
 </ul>
-<h5 id="changes_1">Changes</h5>
+<h5 id="changes_2">Changes</h5>
 <ul>
 <li>Updated dependency on distutils for python 3.12 and above (<a href="https://github.com/rasbt/mlxtend/issues/1072">#1072</a> via <a href="https://github.com/peanutsee">peanutsee</a>)</li>
 </ul>
 <h3 id="version-0230-22-sep-2023">Version 0.23.0 (22 Sep 2023)</h3>
-<h5 id="downloads_2">Downloads</h5>
+<h5 id="downloads_3">Downloads</h5>
 <ul>
 <li>
 <p><a href="https://github.com/rasbt/mlxtend/archive/v0.23.0.zip">Source code (zip)</a></p>
@@ -700,7 +718,7 @@ <h5 id="downloads_2">Downloads</h5>
 <p><a href="https://github.com/rasbt/mlxtend/archive/v0.23.0.tar.gz">Source code (tar.gz)</a></p>
 </li>
 </ul>
-<h5 id="changes_2">Changes</h5>
+<h5 id="changes_3">Changes</h5>
 <ul>
 <li>Address NumPy deprecations to make mlxtend compatible to NumPy 1.24 </li>
 <li>Changed the signature of the <code>LinearRegression</code> model of sklearn in the test removing the <code>normalize</code> parameter as it is deprecated. (<a href="https://github.com/rasbt/mlxtend/issues/1036">#1036</a>)</li>
@@ -709,12 +727,12 @@ <h5 id="changes_2">Changes</h5>
 <li>Converted configuration to <code>pyproject.toml</code> (<a href="https://github.com/rasbt/mlxtend/issues/1065">#1065</a> via <a href="https://github.com/jmahlik">jmahlik</a>)</li>
 <li>Remove <code>mlxtend.image</code>  submodule with face recognition functions due to poor <code>dlib</code> support in modern environments.</li>
 </ul>
-<h5 id="new-features-and-enhancements_1">New Features and Enhancements</h5>
+<h5 id="new-features-and-enhancements_2">New Features and Enhancements</h5>
 <ul>
 <li>Document how to use <code>SequentialFeatureSelector</code> and multiclass ROC AUC. </li>
 </ul>
 <h3 id="version-0220-4-april-2023">Version 0.22.0 (4 April 2023)</h3>
-<h5 id="downloads_3">Downloads</h5>
+<h5 id="downloads_4">Downloads</h5>
 <ul>
 <li>
 <p><a href="https://github.com/rasbt/mlxtend/archive/v0.22.0.zip">Source code (zip)</a></p>
@@ -723,14 +741,14 @@ <h5 id="downloads_3">Downloads</h5>
 <p><a href="https://github.com/rasbt/mlxtend/archive/v0.22.0.tar.gz">Source code (tar.gz)</a></p>
 </li>
 </ul>
-<h5 id="changes_3">Changes</h5>
+<h5 id="changes_4">Changes</h5>
 <ul>
 <li>When <a href="https://rasbt.github.io/mlxtend/user_guide/feature_selection/ExhaustiveFeatureSelector/"><code>ExhaustiveFeatureSelector</code></a> is run with <code>n_jobs == 1</code>, joblib is now disabled, which enables more immediate (live) feedback when the <code>verbose</code> mode is enabled. (<a href="https://github.com/rasbt/mlxtend/pull/985">#985</a> via <a href="https://github.com/NimaSarajpoor">Nima Sarajpoor</a>)</li>
 <li>Disabled unnecessary warning in <a href="https://rasbt.github.io/mlxtend/user_guide/classifier/EnsembleVoteClassifier/"><code>EnsembleVoteClassifier</code></a> (<a href="https://github.com/rasbt/mlxtend/issues/941">#941</a>)</li>
 <li>Fixed various documentation issues (<a href="https://github.com/rasbt/mlxtend/issues/849">#849</a> and <a href="https://github.com/rasbt/mlxtend/issues/951">#951</a> via <a href="https://github.com/zuari1993">Lekshmanan Natarajan</a>)</li>
 <li>Fixed "Edit on GitHub" button  (<a href="https://github.com/rasbt/mlxtend/issues/1024">#1024</a>)</li>
 </ul>
-<h5 id="new-features-and-enhancements_2">New Features and Enhancements</h5>
+<h5 id="new-features-and-enhancements_3">New Features and Enhancements</h5>
 <ul>
 <li>The <a href="https://rasbt.github.io/mlxtend/user_guide/frequent_patterns/association_rules/"><code>mlxtend.frequent_patterns.association_rules</code></a> function has a new metric - Zhang's Metric, which measures both association and dissociation. (<a href="https://github.com/rasbt/mlxtend/pull/980">#980</a>)</li>
 <li>Internal <a href="https://rasbt.github.io/mlxtend/user_guide/frequent_patterns/association_rules/"><code>mlxtend.frequent_patterns.fpmax</code></a> code improvement that avoids casting a sparse DataFrame into a dense NumPy array. (<a href="https://github.com/rasbt/mlxtend/pull/1000">#1000</a> via <a href="https://github.com/tkellogg">Tim Kellogg</a>)</li>
@@ -738,7 +756,7 @@ <h5 id="new-features-and-enhancements_2">New Features and Enhancements</h5>
 <li>Added <a href="https://rasbt.github.io/mlxtend/user_guide/frequent_patterns/hmine/"><code>mlxtend.frequent_patterns.hmine</code></a> algorithm and documentation for mining frequent itemsets using the H-Mine algorithm. (<a href="https://github.com/rasbt/mlxtend/pull/1020">#1020</a> via <a href="https://github.com/fatihsen20">Fatih Sen</a>)</li>
 </ul>
 <h3 id="version-0210-09172022">Version 0.21.0 (09/17/2022)</h3>
-<h5 id="downloads_4">Downloads</h5>
+<h5 id="downloads_5">Downloads</h5>
 <ul>
 <li>
 <p><a href="https://github.com/rasbt/mlxtend/archive/v0.21.0.zip">Source code (zip)</a></p>
@@ -747,12 +765,12 @@ <h5 id="downloads_4">Downloads</h5>
 <p><a href="https://github.com/rasbt/mlxtend/archive/v0.21.0.tar.gz">Source code (tar.gz)</a></p>
 </li>
 </ul>
-<h5 id="new-features-and-enhancements_3">New Features and Enhancements</h5>
+<h5 id="new-features-and-enhancements_4">New Features and Enhancements</h5>
 <ul>
 <li>The <code>mlxtend.evaluate.feature_importance_permutation</code> function has a new <code>feature_groups</code> argument to treat user-specified feature groups as single features, which is useful for one-hot encoded features. (<a href="https://github.com/rasbt/mlxtend/pull/955">#955</a>)</li>
 <li>The <code>mlxtend.feature_selection.ExhaustiveFeatureSelector</code> and <code>SequentialFeatureSelector</code> also gained support for <code>feature_groups</code> with a behavior similar to the one described above.  (<a href="https://github.com/rasbt/mlxtend/pull/957">#957</a> and <a href="https://github.com/rasbt/mlxtend/pull/965">#965</a> via <a href="https://github.com/NimaSarajpoor">Nima Sarajpoor</a>)</li>
 </ul>
-<h5 id="changes_4">Changes</h5>
+<h5 id="changes_5">Changes</h5>
 <ul>
 <li>The <code>custom_feature_names</code> parameter was removed from the <code>ExhaustiveFeatureSelector</code> due to redundancy and to simplify the code base. The <a href="https://rasbt.github.io/mlxtend/user_guide/feature_selection/ExhaustiveFeatureSelector/"><code>ExhaustiveFeatureSelector</code> documentation</a> illustrates how the same behavior and outcome can be achieved using pandas DataFrames.  (<a href="https://github.com/rasbt/mlxtend/pull/957">#957</a>)</li>
 </ul>
@@ -761,7 +779,7 @@ <h5 id="bug-fixes">Bug Fixes</h5>
 <li>None</li>
 </ul>
 <h3 id="version-0200-05262022">Version 0.20.0 (05/26/2022)</h3>
-<h5 id="downloads_5">Downloads</h5>
+<h5 id="downloads_6">Downloads</h5>
 <ul>
 <li>
 <p><a href="https://github.com/rasbt/mlxtend/archive/v0.20.0.zip">Source code (zip)</a></p>
@@ -770,14 +788,14 @@ <h5 id="downloads_5">Downloads</h5>
 <p><a href="https://github.com/rasbt/mlxtend/archive/v0.20.0.tar.gz">Source code (tar.gz)</a></p>
 </li>
 </ul>
-<h5 id="new-features-and-enhancements_4">New Features and Enhancements</h5>
+<h5 id="new-features-and-enhancements_5">New Features and Enhancements</h5>
 <ul>
 <li>The <code>mlxtend.evaluate.bootstrap_point632_score</code> now supports <code>fit_params</code>. (<a href="https://github.com/rasbt/mlxtend/pull/861">#861</a>)</li>
 <li>The <code>mlxtend/plotting/decision_regions.py</code> function now has a <code>contourf_kwargs</code> for matplotlib to change the look of the decision boundaries if desired. (<a href="https://github.com/rasbt/mlxtend/pull/881">#881</a> via [<a href="https://github.com/pbloem">pbloem</a>])</li>
 <li>Add a <code>norm_colormap</code> parameter to <code>mlxtend.plotting.plot_confusion_matrix</code>, to allow normalizing the colormap, e.g., using <code>matplotlib.colors.LogNorm()</code> (<a href="https://github.com/rasbt/mlxtend/pull/895">#895</a>)</li>
 <li>Add new <code>GroupTimeSeriesSplit</code> class for evaluation in time series tasks with support of custom groups and additional parameters in comparison with scikit-learn's <code>TimeSeriesSplit</code>. (<a href="https://github.com/rasbt/mlxtend/pull/915">#915</a> via <a href="https://github.com/labdmitriy">Dmitry Labazkin</a>)</li>
 </ul>
-<h5 id="changes_5">Changes</h5>
+<h5 id="changes_6">Changes</h5>
 <ul>
 <li>Due to compatibility issues with newer package versions, certain functions from six.py have been removed so that mlxtend may not work anymore with Python 2.7.</li>
 <li>As an internal change to speed up unit testing, unit testing is now faciliated by GitHub workflows, and Travis CI and Appveyor hooks have been removed.</li>
@@ -791,7 +809,7 @@ <h5 id="bug-fixes_1">Bug Fixes</h5>
 <li>Fix an issue in <code>mlxtend.plotting.plot_confusion_matrix</code> when string class names are passed (<a href="https://github.com/rasbt/mlxtend/pull/894">#894</a>)</li>
 </ul>
 <h3 id="version-0190-2021-09-02">Version 0.19.0 (2021-09-02)</h3>
-<h5 id="downloads_6">Downloads</h5>
+<h5 id="downloads_7">Downloads</h5>
 <ul>
 <li>
 <p><a href="https://github.com/rasbt/mlxtend/archive/v0.19.0.zip">Source code (zip)</a></p>
@@ -807,7 +825,7 @@ <h5 id="new-features">New Features</h5>
 <li>The <code>evaluate.permutation_test</code> function now accepts a <code>paired</code> argument to specify to support paired permutation/randomization tests. (<a href="https://github.com/rasbt/mlxtend/pull/768">#768</a>)</li>
 <li>The <code>StackingCVRegressor</code> now also supports multi-dimensional targets similar to <code>StackingRegressor</code> via <code>StackingCVRegressor(..., multi_output=True)</code>. (<a href="https://github.com/rasbt/mlxtend/pull/802">#802</a> via <a href="ChromaticIsobar">Marco Tiraboschi</a>)</li>
 </ul>
-<h5 id="changes_6">Changes</h5>
+<h5 id="changes_7">Changes</h5>
 <ul>
 <li>Updates unit tests for scikit-learn 0.24.1 compatibility. (<a href="https://github.com/rasbt/mlxtend/pull/774">#774</a>)</li>
 <li><code>StackingRegressor</code> now requires setting <code>StackingRegressor(..., multi_output=True)</code> if the target is multi-dimensional; this allows for better input validation. (<a href="https://github.com/rasbt/mlxtend/pull/802">#802</a>)</li>
@@ -823,7 +841,7 @@ <h5 id="bug-fixes_2">Bug Fixes</h5>
 <li>Fixes a typo in the SequentialFeatureSelector documentation (<a href="https://github.com/rasbt/mlxtend/issues/835">#835</a> via <a href="https://github.com/joaozanlorensi">João Pedro Zanlorensi Cardoso</a>)</li>
 </ul>
 <h3 id="version-0180-2020-11-25">Version 0.18.0 (2020-11-25)</h3>
-<h5 id="downloads_7">Downloads</h5>
+<h5 id="downloads_8">Downloads</h5>
 <ul>
 <li>
 <p><a href="https://github.com/rasbt/mlxtend/archive/v0.18.0.zip">Source code (zip)</a></p>
@@ -839,7 +857,7 @@ <h5 id="new-features_1">New Features</h5>
 <li>Adds new <code>mlxtend.classifier.OneRClassifier</code> (One Rule Classfier) class, a simple rule-based classifier that is often used as a performance baseline or simple interpretable model. (<a href="https://github.com/rasbt/mlxtend/pull/726">#726</a></li>
 <li>Adds new <code>create_counterfactual</code> method for creating counterfactuals to explain model predictions.  (<a href="https://github.com/rasbt/mlxtend/pull/740">#740</a>)</li>
 </ul>
-<h5 id="changes_7">Changes</h5>
+<h5 id="changes_8">Changes</h5>
 <ul>
 <li><code>permutation_test</code> (<code>mlxtend.evaluate.permutation</code>) ìs corrected to give the proportion of permutations whose statistic is <em>at least as extreme</em> as the one observed. (<a href="https://github.com/rasbt/mlxtend/pull/721">#721</a> via <a href="https://github.com/DarthTrevis">Florian Charlier</a>)</li>
 <li>Fixes the McNemar confusion matrix layout to match the convention (and documentation), swapping the upper left and lower right cells. (<a href="https://github.com/rasbt/mlxtend/pull/744">#744</a> via <a href="https://github.com/mmarius">mmarius</a>)</li>
@@ -850,7 +868,7 @@ <h5 id="bug-fixes_3">Bug Fixes</h5>
 <li>Fixes a bug in <code>bias_variance_decomp</code> where when the <code>mse</code> loss was used, downcasting to integers caused imprecise results for small numbers. (<a href="https://github.com/rasbt/mlxtend/pull/749">#749</a>)</li>
 </ul>
 <h3 id="version-0173-2020-07-27">Version 0.17.3 (2020-07-27)</h3>
-<h5 id="downloads_8">Downloads</h5>
+<h5 id="downloads_9">Downloads</h5>
 <ul>
 <li>
 <p><a href="https://github.com/rasbt/mlxtend/archive/v0.17.3.zip">Source code (zip)</a></p>
@@ -864,7 +882,7 @@ <h5 id="new-features_2">New Features</h5>
 <li>Add <code>predict_proba</code> kwarg to bootstrap methods, to allow bootstrapping of scoring functions that take in probability values. (<a href="https://github.com/rasbt/mlxtend/pull/700">#700</a> via <a href="https://github.com/adam2392">Adam Li</a>)</li>
 <li>Add a <code>cell_values</code> parameter to <code>mlxtend.plotting.heatmap()</code> to optionally suppress cell annotations by setting <code>cell_values=False</code>. (<a href="https://github.com/rasbt/mlxtend/pull/700">#703</a></li>
 </ul>
-<h5 id="changes_8">Changes</h5>
+<h5 id="changes_9">Changes</h5>
 <ul>
 <li>Implemented both <code>use_clones</code> and <code>fit_base_estimators</code> (previously <code>refit</code> in <code>EnsembleVoteClassifier</code>) for <code>EnsembleVoteClassifier</code> and <code>StackingClassifier</code>. (<a href="https://github.com/rasbt/mlxtend/pull/670">#670</a> via <a href="https://github.com/nilichen">Katrina Ni</a>)</li>
 <li>Switched to using raw strings for regex in <code>mlxtend.text</code> to prevent deprecation warning in Python 3.8 (<a href="https://github.com/rasbt/mlxtend/pull/688">#688</a>)</li>
@@ -877,7 +895,7 @@ <h5 id="bug-fixes_4">Bug Fixes</h5>
 <li>Fixes an issue in <code>fpmax</code> that could lead to incorrect support values. (<a href="https://github.com/rasbt/mlxtend/pull/692">#692</a> via <a href="https://github.com/harenbergsd">Steve Harenberg</a>)</li>
 </ul>
 <h3 id="version-0172-2020-02-24">Version 0.17.2 (2020-02-24)</h3>
-<h5 id="downloads_9">Downloads</h5>
+<h5 id="downloads_10">Downloads</h5>
 <ul>
 <li>
 <p><a href="https://github.com/rasbt/mlxtend/archive/v0.17.2.zip">Source code (zip)</a></p>
@@ -890,7 +908,7 @@ <h5 id="new-features_3">New Features</h5>
 <ul>
 <li>-</li>
 </ul>
-<h5 id="changes_9">Changes</h5>
+<h5 id="changes_10">Changes</h5>
 <ul>
 <li>The previously deprecated <code>OnehotTransactions</code> has been removed in favor of the <code>TransactionEncoder.</code></li>
 <li>Removed <code>SparseDataFrame</code> support in frequent pattern mining functions in favor of pandas &gt;=1.0's new way for working sparse data. If you used <code>SparseDataFrame</code> formats, please see pandas' migration guide at https://pandas.pydata.org/pandas-docs/stable/user_guide/sparse.html#migrating. (<a href="https://github.com/rasbt/mlxtend/pull/667">#667</a>)</li>
@@ -901,7 +919,7 @@ <h5 id="bug-fixes_5">Bug Fixes</h5>
 <li>-</li>
 </ul>
 <h3 id="version-0171-2020-01-28">Version 0.17.1 (2020-01-28)</h3>
-<h5 id="downloads_10">Downloads</h5>
+<h5 id="downloads_11">Downloads</h5>
 <ul>
 <li>
 <p><a href="https://github.com/rasbt/mlxtend/archive/v0.17.1.zip">Source code (zip)</a></p>
@@ -916,7 +934,7 @@ <h5 id="new-features_4">New Features</h5>
 <li>Adds a new <code>accuracy_score</code> function to <code>mlxtend.evaluate</code> for computing basic classifcation accuracy, per-class accuracy, and average per-class accuracy. (<a href="https://github.com/rasbt/mlxtend/pull/624">#624</a> via <a href="https://github.com/deepandas11">Deepan Das</a>)</li>
 <li><code>StackingClassifier</code> and <code>StackingCVClassifier</code>now have a <code>decision_function</code> method, which serves as a preferred choice over <code>predict_proba</code> in calculating roc_auc and average_precision scores when the meta estimator is a linear model or support vector classifier. (<a href="https://github.com/rasbt/mlxtend/pull/634">#634</a> via <a href="https://github.com/qiagu">Qiang Gu</a>)</li>
 </ul>
-<h5 id="changes_10">Changes</h5>
+<h5 id="changes_11">Changes</h5>
 <ul>
 <li>Improve the runtime performance for the <code>apriori</code> frequent itemset generating function when <code>low_memory=True</code>. Setting <code>low_memory=False</code> (default) is still faster for small itemsets, but <code>low_memory=True</code> can be much faster for large itemsets and requires less memory.  Also, input validation for  <code>apriori</code>, ̀ fpgrowth<code>and</code>fpmax<code>takes a significant amount of time when input pandas DataFrame is large; this is now dramatically reduced when input contains boolean values (and not zeros/ones), which is the case when using</code>TransactionEncoder`. (<a href="https://github.com/rasbt/mlxtend/pull/619">#619</a> via <a href="https://github.com/dbarbier">Denis Barbier</a>)</li>
 <li>Add support for newer sparse pandas DataFrame for frequent itemset algorithms. Also, input validation for  <code>apriori</code>, ̀ fpgrowth<code>and</code>fpmax` runs much faster on sparse DataFrame when input pandas DataFrame contains integer values. (<a href="https://github.com/rasbt/mlxtend/pull/621">#621</a> via <a href="https://github.com/dbarbier">Denis Barbier</a>)</li>
@@ -932,7 +950,7 @@ <h5 id="bug-fixes_6">Bug Fixes</h5>
 <li>Disable input validation for <code>StackingCVClassifier</code> because it causes issues if pipelines are used as input. <a href="https://github.com/rasbt/mlxtend/pull/606">#606</a></li>
 </ul>
 <h3 id="version-0170-2019-07-19">Version 0.17.0 (2019-07-19)</h3>
-<h5 id="downloads_11">Downloads</h5>
+<h5 id="downloads_12">Downloads</h5>
 <ul>
 <li><a href="https://github.com/rasbt/mlxtend/archive/v0.17.0.zip">Source code (zip)</a></li>
 <li><a href="https://github.com/rasbt/mlxtend/archive/v0.17.0.tar.gz">Source code (tar.gz)</a></li>
@@ -951,7 +969,7 @@ <h5 id="new-features_5">New Features</h5>
 <li>New <code>low_memory</code> option for the <code>apriori</code> frequent itemset generating function. Setting <code>low_memory=False</code> (default) uses a substantially optimized version of the algorithm that is 3-6x faster than the original implementation (<code>low_memory=True</code>). (<a href="https://github.com/rasbt/mlxtend/pull/567">#567</a> via <a href="https://github.com/jmayse">jmayse</a>)</li>
 <li>Added numerically stable OLS methods which uses <code>QR decomposition</code> and <code>Singular Value Decomposition</code> (SVD) methods to <code>LinearRegression</code> in <code>mlxtend.regressor.linear_regression</code>. (<a href="https://github.com/rasbt/mlxtend/pull/575">#575</a> via <a href="https://github.com/PuneetGrov3r">PuneetGrov3r</a>)</li>
 </ul>
-<h5 id="changes_11">Changes</h5>
+<h5 id="changes_12">Changes</h5>
 <ul>
 <li>Now uses the latest joblib library under the hood for multiprocessing instead of <code>sklearn.externals.joblib</code>. (<a href="https://github.com/rasbt/mlxtend/pull/547">#547</a>)</li>
 <li>Changes to <code>StackingCVClassifier</code> and <code>StackingCVRegressor</code> such that first-level models are allowed to generate output of non-numeric type. (<a href="https://github.com/rasbt/mlxtend/pull/562">#562</a>)</li>
@@ -962,7 +980,7 @@ <h5 id="bug-fixes_7">Bug Fixes</h5>
 <li>Make sure that if the <code>'svd'</code> mode is used in PCA, the number of eigenvalues is the same as when using <code>'eigen'</code> (append 0's zeros in that case) (<a href="https://github.com/rasbt/mlxtend/pull/565">#565</a>)</li>
 </ul>
 <h3 id="version-0160-2019-05-12">Version 0.16.0 (2019-05-12)</h3>
-<h5 id="downloads_12">Downloads</h5>
+<h5 id="downloads_13">Downloads</h5>
 <ul>
 <li><a href="https://github.com/rasbt/mlxtend/archive/v0.16.0.zip">Source code (zip)</a></li>
 <li><a href="https://github.com/rasbt/mlxtend/archive/v0.16.0.tar.gz">Source code (tar.gz)</a></li>
@@ -980,7 +998,7 @@ <h5 id="new-features_6">New Features</h5>
 <li>Adds an optional <code>class_name</code> parameter to the confusion matrix function to display class names on the axis as tick marks. (<a href="https://github.com/rasbt/mlxtend/pull/487">#487</a> via <a href="https://github.com/qiaguhttps://github.com/sandpiturtle">sandpiturtle</a>)</li>
 <li>Adds a <code>pca.e_vals_normalized_</code> attribute to PCA for storing the eigenvalues also in normalized form; this is commonly referred to as variance explained ratios. <a href="https://github.com/rasbt/mlxtend/pull/545">#545</a></li>
 </ul>
-<h5 id="changes_12">Changes</h5>
+<h5 id="changes_13">Changes</h5>
 <ul>
 <li>Due to new features, restructuring, and better scikit-learn support (for <code>GridSearchCV</code>, etc.) the <code>StackingCVRegressor</code>'s meta regressor is now being accessed via <code>'meta_regressor__*</code> in the parameter grid. E.g., if a <code>RandomForestRegressor</code> as meta- egressor was previously tuned via <code>'randomforestregressor__n_estimators'</code>, this has now changed to <code>'meta_regressor__n_estimators'</code>. (<a href="https://github.com/rasbt/mlxtend/pull/512">#515</a> via <a href="https://github.com/qiaguhttps://github.com/qiagu">Qiang Gu</a>)</li>
 <li>The same change mentioned above is now applied to other stacking estimators, including <code>StackingClassifier</code>, <code>StackingCVClassifier</code> and <code>StackingRegressor</code>. (<a href="https://github.com/rasbt/mlxtend/pull/522">#522</a> via <a href="https://github.com/qiaguhttps://github.com/qiagu">Qiang Gu</a>)</li>
@@ -995,7 +1013,7 @@ <h5 id="bug-fixes_8">Bug Fixes</h5>
 <li><code>mlxtend.evaluate.feature_importance_permutation</code> now correctly accepts scoring functions with proper function signature as <code>metric</code> argument. <a href="https://github.com/rasbt/mlxtend/pull/528">#528</a></li>
 </ul>
 <h3 id="version-0150-2019-01-19">Version 0.15.0 (2019-01-19)</h3>
-<h5 id="downloads_13">Downloads</h5>
+<h5 id="downloads_14">Downloads</h5>
 <ul>
 <li><a href="https://github.com/rasbt/mlxtend/archive/v0.15.0.zip">Source code (zip)</a></li>
 <li><a href="https://github.com/rasbt/mlxtend/archive/v0.15.0.tar.gz">Source code (tar.gz)</a></li>
@@ -1006,7 +1024,7 @@ <h5 id="new-features_7">New Features</h5>
 <li>Adds a new function, <code>mlxtend.evaluate.bias_variance_decomp</code> that decomposes the loss of a regressor or classifier into bias and variance terms. (<a href="https://github.com/rasbt/mlxtend/pull/470">#470</a>)</li>
 <li>Adds a <code>whitening</code> parameter to <code>PrincipalComponentAnalysis</code>, to optionally whiten the transformed data such that the features have unit variance. (<a href="https://github.com/rasbt/mlxtend/pull/475">#475</a>)</li>
 </ul>
-<h5 id="changes_13">Changes</h5>
+<h5 id="changes_14">Changes</h5>
 <ul>
 <li>Changed the default solver in <code>PrincipalComponentAnalysis</code> to <code>'svd'</code> instead of <code>'eigen'</code> to improve numerical stability. (<a href="https://github.com/rasbt/mlxtend/pull/474">#474</a>)</li>
 <li>The <code>mlxtend.image.extract_face_landmarks</code> now returns <code>None</code> if no facial landmarks were detected instead of an array of all zeros. (<a href="https://github.com/rasbt/mlxtend/pull/466">#466</a>)</li>
@@ -1016,7 +1034,7 @@ <h5 id="bug-fixes_9">Bug Fixes</h5>
 <li>The eigenvectors maybe have not been sorted in certain edge cases if solver was <code>'eigen'</code> in <code>PrincipalComponentAnalysis</code> and <code>LinearDiscriminantAnalysis</code>. (<a href="https://github.com/rasbt/mlxtend/pull/477">#477</a>, <a href="https://github.com/rasbt/mlxtend/pull/478">#478</a>)</li>
 </ul>
 <h3 id="version-0140-2018-11-09">Version 0.14.0 (2018-11-09)</h3>
-<h5 id="downloads_14">Downloads</h5>
+<h5 id="downloads_15">Downloads</h5>
 <ul>
 <li><a href="https://github.com/rasbt/mlxtend/archive/v0.14.0.zip">Source code (zip)</a></li>
 <li><a href="https://github.com/rasbt/mlxtend/archive/v0.14.0.tar.gz">Source code (tar.gz)</a></li>
@@ -1035,7 +1053,7 @@ <h5 id="new-features_8">New Features</h5>
 <li>Added a new <code>mlxtend.evaluate.combined_ftest_5x2cv</code> function to perform an combined 5x2cv F-Test for comparing the performance of two models. (<a href="https://github.com/rasbt/mlxtend/pull/461">#461</a>)</li>
 <li>Added a new <code>mlxtend.evaluate.difference_proportions</code> test for comparing two proportions (e.g., classifier accuracies) (<a href="https://github.com/rasbt/mlxtend/pull/462">#462</a>)</li>
 </ul>
-<h5 id="changes_14">Changes</h5>
+<h5 id="changes_15">Changes</h5>
 <ul>
 <li>Addressed deprecations warnings in NumPy 0.15. (<a href="https://github.com/rasbt/mlxtend/pull/425">#425</a>)</li>
 <li>Because of complications in PR (<a href="https://github.com/rasbt/mlxtend/pull/459">#459</a>), Python 2.7 was now dropped; since official support for Python 2.7 by the Python Software Foundation is ending in approx. 12 months anyways, this re-focussing will hopefully free up some developer time with regard to not having to worry about backward compatibility</li>
@@ -1045,7 +1063,7 @@ <h5 id="bug-fixes_10">Bug Fixes</h5>
 <li>Fixed an issue with a missing import in <code>mlxtend.plotting.plot_confusion_matrix</code>. (<a href="https://github.com/rasbt/mlxtend/pull/428">#428</a>)</li>
 </ul>
 <h3 id="version-0130-2018-07-20">Version 0.13.0 (2018-07-20)</h3>
-<h5 id="downloads_15">Downloads</h5>
+<h5 id="downloads_16">Downloads</h5>
 <ul>
 <li><a href="https://github.com/rasbt/mlxtend/archive/v0.13.0.zip">Source code (zip)</a></li>
 <li><a href="https://github.com/rasbt/mlxtend/archive/v0.13.0.tar.gz">Source code (tar.gz)</a></li>
@@ -1063,7 +1081,7 @@ <h5 id="new-features_9">New Features</h5>
 <li>Added support for merging the meta features with the original input features in <code>StackingRegressor</code> (via <code>use_features_in_secondary</code>) like it is already supported in the other Stacking classes. (<a href="https://github.com/rasbt/mlxtend/pull/418">#418</a>)</li>
 <li>Added a <code>support_only</code> to the <code>association_rules</code> function, which allow constructing association rules (based on the support metric only) for cropped input DataFrames that don't contain a complete set of antecedent and consequent support values. (<a href="https://github.com/rasbt/mlxtend/pull/421">#421</a>)</li>
 </ul>
-<h5 id="changes_15">Changes</h5>
+<h5 id="changes_16">Changes</h5>
 <ul>
 <li>Itemsets generated with <code>apriori</code> are now <code>frozenset</code>s (<a href="https://github.com/rasbt/mlxtend/issues/393">#393</a> by <a href="https://github.com/WLaney">William Laney</a> and <a href="https://github.com/rasbt/mlxtend/issues/394">#394</a>)</li>
 <li>Now raises an error if a input DataFrame to <code>apriori</code> contains non 0, 1, True, False values. <a href="https://github.com/rasbt/mlxtend/issues/419">#419</a>)</li>
@@ -1077,7 +1095,7 @@ <h5 id="bug-fixes_11">Bug Fixes</h5>
 <li>Allow <code>StackingCVClassifier</code> to work with sparse matrices when <code>use_features_in_secondary=True</code>  (<a href="https://github.com/rasbt/mlxtend/issues/417">#417</a>)</li>
 </ul>
 <h3 id="version-0120-2018-21-04">Version 0.12.0 (2018-21-04)</h3>
-<h5 id="downloads_16">Downloads</h5>
+<h5 id="downloads_17">Downloads</h5>
 <ul>
 <li><a href="https://github.com/rasbt/mlxtend/archive/v0.12.0.zip">Source code (zip)</a></li>
 <li><a href="https://github.com/rasbt/mlxtend/archive/v0.12.0.tar.gz">Source code (tar.gz)</a></li>
@@ -1089,7 +1107,7 @@ <h5 id="new-features_10">New Features</h5>
 <li>The fit method of the <code>SequentialFeatureSelector</code> now optionally accepts
 <code>**fit_params</code> for the estimator that is used for the feature selection. (<a href="https://github.com/rasbt/mlxtend/pull/350">#350</a> by Zach Griffith)</li>
 </ul>
-<h5 id="changes_16">Changes</h5>
+<h5 id="changes_17">Changes</h5>
 <ul>
 <li>Replaced <code>plot_decision_regions</code> colors by a colorblind-friendly palette and adds contour lines for decision regions. (<a href="https://github.com/rasbt/mlxtend/issues/348">#348</a>)</li>
 <li>All stacking estimators now raise <code>NonFittedErrors</code> if any method for inference is called prior to fitting the estimator. (<a href="https://github.com/rasbt/mlxtend/issues/353">#353</a>)</li>
@@ -1102,7 +1120,7 @@ <h5 id="bug-fixes_12">Bug Fixes</h5>
 <li>Many documentation improvements, including links to the User Guides in the API docs (<a href="https://github.com/rasbt/mlxtend/pull/371">#371</a>)</li>
 </ul>
 <h3 id="version-0110-2018-03-14">Version 0.11.0 (2018-03-14)</h3>
-<h5 id="downloads_17">Downloads</h5>
+<h5 id="downloads_18">Downloads</h5>
 <ul>
 <li><a href="https://github.com/rasbt/mlxtend/archive/v0.11.0.zip">Source code (zip)</a></li>
 <li><a href="https://github.com/rasbt/mlxtend/archive/v0.11.0.tar.gz">Source code (tar.gz)</a></li>
@@ -1119,7 +1137,7 @@ <h5 id="new-features_11">New Features</h5>
 <li>A <code>refit</code> parameter was added to stacking classes (similar to the <code>refit</code> parameter in the <code>EnsembleVoteClassifier</code>), to support classifiers and regressors that follow the scikit-learn API but are not compatible with scikit-learn's <code>clone</code> function. (<a href="https://github.com/rasbt/mlxtend/issues/322">#322</a>)</li>
 <li>The <code>ColumnSelector</code> now has a <code>drop_axis</code> argument to use it in pipelines with <code>CountVectorizers</code>. (<a href="https://github.com/rasbt/mlxtend/pull/333">#333</a>)</li>
 </ul>
-<h5 id="changes_17">Changes</h5>
+<h5 id="changes_18">Changes</h5>
 <ul>
 <li>Raises an informative error message if <code>predict</code> or <code>predict_meta_features</code> is called prior to calling the <code>fit</code> method in <code>StackingRegressor</code> and <code>StackingCVRegressor</code>. (<a href="https://github.com/rasbt/mlxtend/issues/315">#315</a>)</li>
 <li>The <code>plot_decision_regions</code> function now automatically determines the optimal setting based on the feature dimensions and supports anti-aliasing. The old <code>res</code>  parameter has been deprecated. (<a href="https://github.com/rasbt/mlxtend/pull/309">#309</a> by <a href="https://github.com/arteymix">Guillaume Poirier-Morency</a>)</li>
@@ -1135,7 +1153,7 @@ <h5 id="bug-fixes_13">Bug Fixes</h5>
 <li>Fixed bug that raised an indexing error if the number of items was &lt;= 1 when computing association rules using the conviction metric. (<a href="https://github.com/rasbt/mlxtend/issues/340">#340</a>)</li>
 </ul>
 <h3 id="version-0100-2017-12-22">Version 0.10.0 (2017-12-22)</h3>
-<h5 id="downloads_18">Downloads</h5>
+<h5 id="downloads_19">Downloads</h5>
 <ul>
 <li><a href="https://github.com/rasbt/mlxtend/archive/v0.10.0.zip">Source code (zip)</a></li>
 <li><a href="https://github.com/rasbt/mlxtend/archive/v0.10.0.tar.gz">Source code (tar.gz)</a></li>
@@ -1148,7 +1166,7 @@ <h5 id="new-features_12">New Features</h5>
 <li>New function (<code>evaluate.mcnemar_tables</code>) for creating multiple 2x2 contigency from model predictions arrays that can be used in multiple McNemar (post-hoc) tests or Cochran's Q or F tests, etc. (<a href="https://github.com/rasbt/mlxtend/issues/307">#307</a>)</li>
 <li>New function (<code>evaluate.cochrans_q</code>) for performing Cochran's Q test to compare the accuracy of multiple classifiers. (<a href="https://github.com/rasbt/mlxtend/issues/310">#310</a>)</li>
 </ul>
-<h5 id="changes_18">Changes</h5>
+<h5 id="changes_19">Changes</h5>
 <ul>
 <li>Added <code>requirements.txt</code> to <code>setup.py</code>. (<a href="https://github.com/rasbt/mlxtend/issues/304">#304</a> via <a href="https://github.com/ColCarroll">Colin Carrol</a>)</li>
 </ul>
@@ -1158,7 +1176,7 @@ <h5 id="bug-fixes_14">Bug Fixes</h5>
 <li><code>nose</code> is not required to use the library (<a href="https://github.com/rasbt/mlxtend/issues/302">#302</a>)</li>
 </ul>
 <h3 id="version-091-2017-11-19">Version 0.9.1 (2017-11-19)</h3>
-<h5 id="downloads_19">Downloads</h5>
+<h5 id="downloads_20">Downloads</h5>
 <ul>
 <li><a href="https://github.com/rasbt/mlxtend/archive/v0.9.1.zip">Source code (zip)</a></li>
 <li><a href="https://github.com/rasbt/mlxtend/archive/v0.9.1.tar.gz">Source code (tar.gz)</a></li>
@@ -1168,7 +1186,7 @@ <h5 id="new-features_13">New Features</h5>
 <li>Added <code>mlxtend.evaluate.bootstrap_point632_score</code> to evaluate the performance of estimators using the .632 bootstrap. (<a href="https://github.com/rasbt/mlxtend/pull/283">#283</a>)</li>
 <li>New <code>max_len</code> parameter for the frequent itemset generation via the <code>apriori</code> function to allow for early stopping. (<a href="https://github.com/rasbt/mlxtend/pull/270">#270</a>)</li>
 </ul>
-<h5 id="changes_19">Changes</h5>
+<h5 id="changes_20">Changes</h5>
 <ul>
 <li>All feature index tuples in <code>SequentialFeatureSelector</code> or now in sorted order. (<a href="https://github.com/rasbt/mlxtend/pull/262">#262</a>)</li>
 <li>The <code>SequentialFeatureSelector</code> now runs the continuation of the floating inclusion/exclusion as described in Novovicova &amp; Kittler (1994).
@@ -1181,7 +1199,7 @@ <h5 id="bug-fixes_15">Bug Fixes</h5>
 <li>Fixed an deprecation error that occured with McNemar test when using SciPy 1.0. (<a href="https://github.com/rasbt/mlxtend/pull/283">#283</a>)</li>
 </ul>
 <h3 id="version-090-2017-10-21">Version 0.9.0 (2017-10-21)</h3>
-<h5 id="downloads_20">Downloads</h5>
+<h5 id="downloads_21">Downloads</h5>
 <ul>
 <li><a href="https://github.com/rasbt/mlxtend/archive/v0.9.0.zip">Source code (zip)</a></li>
 <li><a href="https://github.com/rasbt/mlxtend/archive/v0.9.0.tar.gz">Source code (tar.gz)</a></li>
@@ -1196,7 +1214,7 @@ <h5 id="new-features_14">New Features</h5>
 <li>Added a new <code>BootstrapOutOfBag</code> class, an implementation of the out-of-bag bootstrap to evaluate supervised learning algorithms. (<a href="https://github.com/rasbt/mlxtend/pull/265">#265</a>)</li>
 <li>The parameters for <code>StackingClassifier</code>, <code>StackingCVClassifier</code>, <code>StackingRegressor</code>, <code>StackingCVRegressor</code>, and <code>EnsembleVoteClassifier</code> can now be tuned using scikit-learn's <code>GridSearchCV</code> (<a href="https://github.com/rasbt/mlxtend/pull/254">#254</a> via <a href="https://github.com/jrbourbeau">James Bourbeau</a>)</li>
 </ul>
-<h5 id="changes_20">Changes</h5>
+<h5 id="changes_21">Changes</h5>
 <ul>
 <li>The <code>'support'</code> column returned by <code>frequent_patterns.association_rules</code> was changed to compute the support of "antecedant union consequent", and new <code>antecedant support'</code> and <code>'consequent support'</code> column were added to avoid ambiguity. (<a href="https://github.com/rasbt/mlxtend/pull/245">#245</a>)</li>
 <li>Allow the <code>OnehotTransactions</code> to be cloned via scikit-learn's <code>clone</code> function, which is required by e.g., scikit-learn's <code>FeatureUnion</code> or <code>GridSearchCV</code> (via <a href="https://github.com/iaroslav-ai">Iaroslav Shcherbatyi</a>). (<a href="https://github.com/rasbt/mlxtend/pull/249">#249</a>)</li>
@@ -1208,7 +1226,7 @@ <h5 id="bug-fixes_16">Bug Fixes</h5>
 <li>The vectors from SVD in <code>PrincipalComponentAnalysis</code> are now being scaled so that the eigenvalues via <code>solver='eigen'</code> and <code>solver='svd'</code> now store eigenvalues that have the same magnitudes. (<a href="https://github.com/rasbt/mlxtend/pull/251">#251</a>)</li>
 </ul>
 <h3 id="version-080-2017-09-09">Version 0.8.0 (2017-09-09)</h3>
-<h5 id="downloads_21">Downloads</h5>
+<h5 id="downloads_22">Downloads</h5>
 <ul>
 <li><a href="https://github.com/rasbt/mlxtend/archive/v0.8.0.zip">Source code (zip)</a></li>
 <li><a href="https://github.com/rasbt/mlxtend/archive/v0.8.0.tar.gz">Source code (tar.gz)</a></li>
@@ -1218,7 +1236,7 @@ <h5 id="new-features_15">New Features</h5>
 <li>Added a <code>mlxtend.evaluate.bootstrap</code> that implements the ordinary nonparametric bootstrap to bootstrap a single statistic (for example, the mean. median, R^2 of a regression fit, and so forth) <a href="https://github.com/rasbt/mlxtend/pull/232">#232</a></li>
 <li><code>SequentialFeatureSelecor</code>'s <code>k_features</code> now accepts a string argument "best" or "parsimonious" for more "automated" feature selection. For instance, if "best" is provided, the feature selector will return the feature subset with the best cross-validation performance. If "parsimonious" is provided as an argument, the smallest feature subset that is within one standard error of the cross-validation performance will be selected. <a href="https://github.com/rasbt/mlxtend/pull/238">#238</a></li>
 </ul>
-<h5 id="changes_21">Changes</h5>
+<h5 id="changes_22">Changes</h5>
 <ul>
 <li><code>SequentialFeatureSelector</code> now uses <code>np.nanmean</code> over normal mean to support scorers that may return <code>np.nan</code>  <a href="https://github.com/rasbt/mlxtend/pull/211">#211</a> (via <a href="https://github.com/mrkaiser">mrkaiser</a>)</li>
 <li>The <code>skip_if_stuck</code> parameter was removed from <code>SequentialFeatureSelector</code> in favor of a more efficient implementation comparing the conditional inclusion/exclusion results (in the floating versions) to the performances of previously sampled feature sets that were cached <a href="https://github.com/rasbt/mlxtend/pull/237">#237</a></li>
@@ -1229,7 +1247,7 @@ <h5 id="bug-fixes_17">Bug Fixes</h5>
 <li>Fixed a bug where the <code>SequentialFeatureSelector</code> selected a feature subset larger than then specified via the <code>k_features</code> tuple max-value <a href="https://github.com/rasbt/mlxtend/pull/213">#213</a></li>
 </ul>
 <h3 id="version-070-2017-06-22">Version 0.7.0 (2017-06-22)</h3>
-<h5 id="downloads_22">Downloads</h5>
+<h5 id="downloads_23">Downloads</h5>
 <ul>
 <li><a href="https://github.com/rasbt/mlxtend/archive/v0.7.0.zip">Source code (zip)</a></li>
 <li><a href="https://github.com/rasbt/mlxtend/archive/v0.7.0.tar.gz">Source code (tar.gz)</a></li>
@@ -1239,7 +1257,7 @@ <h5 id="new-features_16">New Features</h5>
 <li>New <a href="https://rasbt.github.io/mlxtend/user_guide/plotting/ecdf/">mlxtend.plotting.ecdf</a> function for plotting empirical cumulative distribution functions (<a href="https://github.com/rasbt/mlxtend/pull/196">#196</a>).</li>
 <li>New <a href="https://rasbt.github.io/mlxtend/user_guide/regressor/StackingCVRegressor/"><code>StackingCVRegressor</code></a> for stacking regressors with out-of-fold predictions to prevent overfitting (<a href="https://github.com/rasbt/mlxtend/pull/201">#201</a>via <a href="https://github.com/EikeDehling">Eike Dehling</a>).</li>
 </ul>
-<h5 id="changes_22">Changes</h5>
+<h5 id="changes_23">Changes</h5>
 <ul>
 <li>The TensorFlow estimator have been removed from mlxtend, since TensorFlow has now very convenient ways to build on estimators, which render those implementations obsolete.</li>
 <li><code>plot_decision_regions</code> now supports plotting decision regions for more than 2 training features <a href="https://github.com/rasbt/mlxtend/pull/189">#189</a>, via <a href="https://github.com/jrbourbeau">James Bourbeau</a>).</li>
@@ -1254,7 +1272,7 @@ <h5 id="bug-fixes_18">Bug Fixes</h5>
 <li>Made <code>math.num_combinations</code> and <code>math.num_permutations</code> numerically stable for large numbers of combinations and permutations (<a href="https://github.com/rasbt/mlxtend/pull/200">#200</a>).</li>
 </ul>
 <h3 id="version-060-2017-03-18">Version 0.6.0 (2017-03-18)</h3>
-<h5 id="downloads_23">Downloads</h5>
+<h5 id="downloads_24">Downloads</h5>
 <ul>
 <li><a href="https://github.com/rasbt/mlxtend/archive/v0.6.0.zip">Source code (zip)</a></li>
 <li><a href="https://github.com/rasbt/mlxtend/archive/v0.6.0.tar.gz">Source code (tar.gz)</a></li>
@@ -1263,7 +1281,7 @@ <h5 id="new-features_17">New Features</h5>
 <ul>
 <li>An <code>association_rules</code> function is implemented that allows to generate rules based on a list of frequent itemsets (via <a href="https://github.com/JoshuaGoerner">Joshua Goerner</a>).</li>
 </ul>
-<h5 id="changes_23">Changes</h5>
+<h5 id="changes_24">Changes</h5>
 <ul>
 <li>Adds a black <code>edgecolor</code> to plots via <code>plotting.plot_decision_regions</code> to make markers more distinguishable from the background in <code>matplotlib&gt;=2.0</code>.</li>
 <li>The <code>association</code> submodule was renamed to <code>frequent_patterns</code>.</li>
@@ -1274,7 +1292,7 @@ <h5 id="bug-fixes_19">Bug Fixes</h5>
 <li>Fixed typos in autompg and wine datasets (via <a href="https://github.com/jrbourbeau">James Bourbeau</a>).</li>
 </ul>
 <h3 id="version-051-2017-02-14">Version 0.5.1 (2017-02-14)</h3>
-<h5 id="downloads_24">Downloads</h5>
+<h5 id="downloads_25">Downloads</h5>
 <ul>
 <li><a href="https://github.com/rasbt/mlxtend/archive/v0.5.1.zip">Source code (zip)</a></li>
 <li><a href="https://github.com/rasbt/mlxtend/archive/v0.5.1.tar.gz">Source code (tar.gz)</a></li>
@@ -1286,7 +1304,7 @@ <h5 id="new-features_18">New Features</h5>
 <li><code>StackingClassifier</code> and <code>StackingRegressor</code> support multivariate targets if the underlying models do (via <a href="https://github.com/kernc">kernc</a>).</li>
 <li><code>StackingClassifier</code> has a new <code>use_features_in_secondary</code> attribute like <code>StackingCVClassifier</code>.</li>
 </ul>
-<h5 id="changes_24">Changes</h5>
+<h5 id="changes_25">Changes</h5>
 <ul>
 <li>Changed default verbosity level in <code>SequentialFeatureSelector</code> to 0</li>
 <li>The <code>EnsembleVoteClassifier</code> now raises a <code>NotFittedError</code> if the estimator wasn't <code>fit</code> before calling <code>predict</code>. (via <a href="https://github.com/avloss">Anton Loss</a>)</li>
@@ -1300,7 +1318,7 @@ <h5 id="bug-fixes_20">Bug Fixes</h5>
 <li>Fixed a bug that could occur in the <code>SequentialFeatureSelector</code> if there are similarly-well performing subsets in the floating variants (via <a href="https://github.com/zacwellmer">Zac Wellmer</a>).</li>
 </ul>
 <h3 id="version-050-2016-11-09">Version 0.5.0 (2016-11-09)</h3>
-<h5 id="downloads_25">Downloads</h5>
+<h5 id="downloads_26">Downloads</h5>
 <ul>
 <li><a href="https://github.com/rasbt/mlxtend/archive/v0.5.0.zip">Source code (zip)</a></li>
 <li><a href="https://github.com/rasbt/mlxtend/archive/v0.5.0.tar.gz">Source code (tar.gz)</a></li>
@@ -1316,7 +1334,7 @@ <h5 id="new-features_19">New Features</h5>
 <li>New <code>checkerboard_plot</code> function in <code>plotting</code> to plot checkerboard tables / heat maps</li>
 <li>New <code>mcnemar_table</code> and <code>mcnemar</code> functions in <code>evaluate</code> to compute 2x2 contingency tables and McNemar's test</li>
 </ul>
-<h5 id="changes_25">Changes</h5>
+<h5 id="changes_26">Changes</h5>
 <ul>
 <li>All plotting functions have been moved to <code>mlxtend.plotting</code> for compatibility reasons with continuous integration services and to make the installation of <code>matplotlib</code> optional for users of <code>mlxtend</code>'s core functionality</li>
 <li>Added a compatibility layer for <code>scikit-learn 0.18</code> using the new <code>model_selection</code> module  while maintaining backwards compatibility to scikit-learn 0.17.</li>
@@ -1327,7 +1345,7 @@ <h5 id="bug-fixes_21">Bug Fixes</h5>
 <li>Raise <code>AttributeError</code> in <code>plot_decision_regions</code> when the <code>X_higlight</code> argument is a 1D array (<a href="https://github.com/chkoar">chkoar</a>)</li>
 </ul>
 <h3 id="version-042-2016-08-24">Version 0.4.2 (2016-08-24)</h3>
-<h5 id="downloads_26">Downloads</h5>
+<h5 id="downloads_27">Downloads</h5>
 <ul>
 <li><a href="https://github.com/rasbt/mlxtend/archive/v0.4.2.zip">Source code (zip)</a></li>
 <li><a href="https://github.com/rasbt/mlxtend/archive/v0.4.2.tar.gz">Source code (tar.gz)</a></li>
@@ -1338,7 +1356,7 @@ <h5 id="new-features_20">New Features</h5>
 <li>Added <code>preprocessing.CopyTransformer</code>, a mock class that returns copies of
 imput arrays via <code>transform</code> and <code>fit_transform</code></li>
 </ul>
-<h5 id="changes_26">Changes</h5>
+<h5 id="changes_27">Changes</h5>
 <ul>
 <li>Added AppVeyor to CI to ensure MS Windows compatibility</li>
 <li>Dataset are now saved as compressed .txt or .csv files rather than being imported as Python objects</li>
@@ -1359,7 +1377,7 @@ <h5 id="bug-fixes_22">Bug Fixes</h5>
 <li>Fixed rare bug in MLP <code>_layer_mapping</code> functions that caused a swap between the random number generation seed when initializing weights and biases</li>
 </ul>
 <h3 id="version-041-2016-05-01">Version 0.4.1 (2016-05-01)</h3>
-<h5 id="downloads_27">Downloads</h5>
+<h5 id="downloads_28">Downloads</h5>
 <ul>
 <li><a href="https://github.com/rasbt/mlxtend/archive/v0.4.1.zip">Source code (zip)</a></li>
 <li><a href="https://github.com/rasbt/mlxtend/archive/v0.4.1.tar.gz">Source code (tar.gz)</a></li>
@@ -1371,7 +1389,7 @@ <h5 id="new-features_21">New Features</h5>
 <li>New k-means clustering estimator (<a href="../user_guide/cluster/Kmeans/"><code>cluster.Kmeans</code></a>)</li>
 <li>New TensorFlow k-means clustering estimator (<code>tf_cluster.Kmeans</code>)</li>
 </ul>
-<h5 id="changes_27">Changes</h5>
+<h5 id="changes_28">Changes</h5>
 <ul>
 <li>Due to refactoring of the estimator classes, the <code>init_weights</code> parameter of the <code>fit</code> methods was globally renamed to <code>init_params</code></li>
 <li>Overall performance improvements of estimators due to code clean-up and refactoring</li>
@@ -1403,12 +1421,12 @@ <h5 id="new-features_22">New Features</h5>
 <li>New RBF Kernel Principal Component Analysis class via <a href="../user_guide/feature_extraction/RBFKernelPCA/"><code>mlxtend.feature_extraction.RBFKernelPCA</code></a></li>
 <li>New Linear Discriminant Analysis class via <a href="../user_guide/feature_extraction/LinearDiscriminantAnalysis/"><code>mlxtend.feature_extraction.LinearDiscriminantAnalysis</code></a></li>
 </ul>
-<h5 id="changes_28">Changes</h5>
+<h5 id="changes_29">Changes</h5>
 <ul>
 <li>The <code>column</code> parameter in <code>mlxtend.preprocessing.standardize</code> now defaults to <code>None</code> to standardize all columns more conveniently</li>
 </ul>
 <h3 id="version-030-2016-01-31">Version 0.3.0 (2016-01-31)</h3>
-<h5 id="downloads_28">Downloads</h5>
+<h5 id="downloads_29">Downloads</h5>
 <ul>
 <li><a href="https://github.com/rasbt/mlxtend/archive/v0.3.0.zip">Source code (zip)</a></li>
 <li><a href="https://github.com/rasbt/mlxtend/archive/v0.3.0.tar.gz">Source code (tar.gz)</a></li>
@@ -1426,7 +1444,7 @@ <h5 id="new-features_23">New Features</h5>
 <li>Added Boston <code>housing dataset</code></li>
 <li>New <code>shuffle</code> parameter for <code>classifier.NeuralNetMLP</code></li>
 </ul>
-<h5 id="changes_29">Changes</h5>
+<h5 id="changes_30">Changes</h5>
 <ul>
 <li>The <code>mlxtend.preprocessing.standardize</code> function now optionally returns the parameters, which are estimated from the array, for re-use. A further improvement makes the <code>standardize</code> function smarter in order to avoid zero-division errors</li>
 <li>Cosmetic improvements to the <code>evaluate.plot_decision_regions</code> function such as hiding plot axes</li>
@@ -1439,7 +1457,7 @@ <h5 id="changes_29">Changes</h5>
 <li>Renaming <code>mlxtend.plotting</code> to <code>mlxtend.general_plotting</code> in order to distinguish general plotting function from specialized utility function such as <code>evaluate.plot_decision_regions</code></li>
 </ul>
 <h3 id="version-029-2015-07-14">Version 0.2.9 (2015-07-14)</h3>
-<h5 id="downloads_29">Downloads</h5>
+<h5 id="downloads_30">Downloads</h5>
 <ul>
 <li><a href="https://github.com/rasbt/mlxtend/archive/v0.2.9.zip">Source code (zip)</a></li>
 <li><a href="https://github.com/rasbt/mlxtend/archive/v0.2.9.tar.gz">Source code (tar.gz)</a></li>
@@ -1448,7 +1466,7 @@ <h5 id="new-features_24">New Features</h5>
 <ul>
 <li>Sequential Feature Selection algorithms: SFS, SFFS, SBS, and SFBS</li>
 </ul>
-<h5 id="changes_30">Changes</h5>
+<h5 id="changes_31">Changes</h5>
 <ul>
 <li>Changed <code>regularization</code> &amp; <code>lambda</code> parameters in <code>LogisticRegression</code> to single parameter <code>l2_lambda</code></li>
 </ul>
@@ -1586,14 +1604,16 @@ <h3 id="version-011-2014-08-13">Version 0.1.1 (2014-08-13)</h3>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../js/jquery-3.6.0.min.js"></script>
+        <script src="../js/bootstrap.min.js"></script>
         <script>
             var base_url = "..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../mathjaxhelper.js" defer></script>
-        <script src="../search/main.js" defer></script>
+        <script src="../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../mathjaxhelper.js"></script>
+        <script src="../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/CONTRIBUTING/index.html b/CONTRIBUTING/index.html
index d13be518f..bd6bb6e92 100644
--- a/CONTRIBUTING/index.html
+++ b/CONTRIBUTING/index.html
@@ -12,25 +12,22 @@
         <link href="../css/bootstrap.min.css" rel="stylesheet">
         <link href="../css/font-awesome.min.css" rel="stylesheet">
         <link href="../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../cinder/css/base.css" rel="stylesheet">
         <link href="../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../cinder/css/cinder.css" rel="stylesheet">
         <link href="../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -980,14 +977,16 @@ <h3 id="5-updating-the-conda-forge-recipe">5. Updating the conda-forge recipe</h
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../js/jquery-3.6.0.min.js"></script>
+        <script src="../js/bootstrap.min.js"></script>
         <script>
             var base_url = "..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../mathjaxhelper.js" defer></script>
-        <script src="../search/main.js" defer></script>
+        <script src="../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../mathjaxhelper.js"></script>
+        <script src="../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/Code-of-Conduct/index.html b/Code-of-Conduct/index.html
index d2fa7186d..701a0876b 100644
--- a/Code-of-Conduct/index.html
+++ b/Code-of-Conduct/index.html
@@ -12,25 +12,22 @@
         <link href="../css/bootstrap.min.css" rel="stylesheet">
         <link href="../css/font-awesome.min.css" rel="stylesheet">
         <link href="../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../cinder/css/base.css" rel="stylesheet">
         <link href="../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../cinder/css/cinder.css" rel="stylesheet">
         <link href="../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -721,14 +718,16 @@ <h2 id="attribution">Attribution</h2>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../js/jquery-3.6.0.min.js"></script>
+        <script src="../js/bootstrap.min.js"></script>
         <script>
             var base_url = "..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../mathjaxhelper.js" defer></script>
-        <script src="../search/main.js" defer></script>
+        <script src="../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../mathjaxhelper.js"></script>
+        <script src="../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/USER-GUIDE-INDEX/index.html b/USER-GUIDE-INDEX/index.html
index 84cf45707..57b654b1e 100644
--- a/USER-GUIDE-INDEX/index.html
+++ b/USER-GUIDE-INDEX/index.html
@@ -12,25 +12,22 @@
         <link href="../css/bootstrap.min.css" rel="stylesheet">
         <link href="../css/font-awesome.min.css" rel="stylesheet">
         <link href="../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../cinder/css/base.css" rel="stylesheet">
         <link href="../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../cinder/css/cinder.css" rel="stylesheet">
         <link href="../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -809,14 +806,16 @@ <h2 id="utils"><code>utils</code></h2>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../js/jquery-3.6.0.min.js"></script>
+        <script src="../js/bootstrap.min.js"></script>
         <script>
             var base_url = "..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../mathjaxhelper.js" defer></script>
-        <script src="../search/main.js" defer></script>
+        <script src="../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../mathjaxhelper.js"></script>
+        <script src="../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/USER_GUIDE_INDEX/index.html b/USER_GUIDE_INDEX/index.html
index fa5b223ce..5f865012c 100644
--- a/USER_GUIDE_INDEX/index.html
+++ b/USER_GUIDE_INDEX/index.html
@@ -12,25 +12,22 @@
         <link href="../css/bootstrap.min.css" rel="stylesheet">
         <link href="../css/font-awesome.min.css" rel="stylesheet">
         <link href="../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../cinder/css/base.css" rel="stylesheet">
         <link href="../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../cinder/css/cinder.css" rel="stylesheet">
         <link href="../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -821,14 +818,16 @@ <h2 id="utils"><code>utils</code></h2>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../js/jquery-3.6.0.min.js"></script>
+        <script src="../js/bootstrap.min.js"></script>
         <script>
             var base_url = "..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../mathjaxhelper.js" defer></script>
-        <script src="../search/main.js" defer></script>
+        <script src="../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../mathjaxhelper.js"></script>
+        <script src="../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/api_modules/mlxtend.classifier/Adaline/index.html b/api_modules/mlxtend.classifier/Adaline/index.html
index fe24af6f6..7abf2dcc4 100644
--- a/api_modules/mlxtend.classifier/Adaline/index.html
+++ b/api_modules/mlxtend.classifier/Adaline/index.html
@@ -12,25 +12,22 @@
         <link href="../../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../../cinder/css/base.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -792,14 +789,16 @@ <h3 id="methods">Methods</h3>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../../mathjaxhelper.js" defer></script>
-        <script src="../../../search/main.js" defer></script>
+        <script src="../../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../../mathjaxhelper.js"></script>
+        <script src="../../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/api_modules/mlxtend.classifier/EnsembleVoteClassifier/index.html b/api_modules/mlxtend.classifier/EnsembleVoteClassifier/index.html
index df2e6c60d..9e0d7716d 100644
--- a/api_modules/mlxtend.classifier/EnsembleVoteClassifier/index.html
+++ b/api_modules/mlxtend.classifier/EnsembleVoteClassifier/index.html
@@ -12,25 +12,22 @@
         <link href="../../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../../cinder/css/base.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -934,14 +931,16 @@ <h3 id="methods">Methods</h3>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../../mathjaxhelper.js" defer></script>
-        <script src="../../../search/main.js" defer></script>
+        <script src="../../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../../mathjaxhelper.js"></script>
+        <script src="../../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/api_modules/mlxtend.classifier/LogisticRegression/index.html b/api_modules/mlxtend.classifier/LogisticRegression/index.html
index 0be2c153a..a430cc836 100644
--- a/api_modules/mlxtend.classifier/LogisticRegression/index.html
+++ b/api_modules/mlxtend.classifier/LogisticRegression/index.html
@@ -12,25 +12,22 @@
         <link href="../../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../../cinder/css/base.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -813,14 +810,16 @@ <h3 id="methods">Methods</h3>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../../mathjaxhelper.js" defer></script>
-        <script src="../../../search/main.js" defer></script>
+        <script src="../../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../../mathjaxhelper.js"></script>
+        <script src="../../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/api_modules/mlxtend.classifier/MultiLayerPerceptron/index.html b/api_modules/mlxtend.classifier/MultiLayerPerceptron/index.html
index d00475134..0d698e359 100644
--- a/api_modules/mlxtend.classifier/MultiLayerPerceptron/index.html
+++ b/api_modules/mlxtend.classifier/MultiLayerPerceptron/index.html
@@ -12,25 +12,22 @@
         <link href="../../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../../cinder/css/base.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -837,14 +834,16 @@ <h3 id="methods">Methods</h3>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../../mathjaxhelper.js" defer></script>
-        <script src="../../../search/main.js" defer></script>
+        <script src="../../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../../mathjaxhelper.js"></script>
+        <script src="../../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/api_modules/mlxtend.classifier/OneRClassifier/index.html b/api_modules/mlxtend.classifier/OneRClassifier/index.html
index a9e4026c2..b4705084d 100644
--- a/api_modules/mlxtend.classifier/OneRClassifier/index.html
+++ b/api_modules/mlxtend.classifier/OneRClassifier/index.html
@@ -12,25 +12,22 @@
         <link href="../../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../../cinder/css/base.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -785,14 +782,16 @@ <h3 id="methods">Methods</h3>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../../mathjaxhelper.js" defer></script>
-        <script src="../../../search/main.js" defer></script>
+        <script src="../../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../../mathjaxhelper.js"></script>
+        <script src="../../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/api_modules/mlxtend.classifier/Perceptron/index.html b/api_modules/mlxtend.classifier/Perceptron/index.html
index 6f8f8afae..a19b5c617 100644
--- a/api_modules/mlxtend.classifier/Perceptron/index.html
+++ b/api_modules/mlxtend.classifier/Perceptron/index.html
@@ -12,25 +12,22 @@
         <link href="../../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../../cinder/css/base.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -783,14 +780,16 @@ <h3 id="methods">Methods</h3>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../../mathjaxhelper.js" defer></script>
-        <script src="../../../search/main.js" defer></script>
+        <script src="../../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../../mathjaxhelper.js"></script>
+        <script src="../../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/api_modules/mlxtend.classifier/SoftmaxRegression/index.html b/api_modules/mlxtend.classifier/SoftmaxRegression/index.html
index e107dbed0..06a83174b 100644
--- a/api_modules/mlxtend.classifier/SoftmaxRegression/index.html
+++ b/api_modules/mlxtend.classifier/SoftmaxRegression/index.html
@@ -12,25 +12,22 @@
         <link href="../../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../../cinder/css/base.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -815,14 +812,16 @@ <h3 id="methods">Methods</h3>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../../mathjaxhelper.js" defer></script>
-        <script src="../../../search/main.js" defer></script>
+        <script src="../../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../../mathjaxhelper.js"></script>
+        <script src="../../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/api_modules/mlxtend.classifier/StackingCVClassifier/index.html b/api_modules/mlxtend.classifier/StackingCVClassifier/index.html
index bef66b35b..a98782fe0 100644
--- a/api_modules/mlxtend.classifier/StackingCVClassifier/index.html
+++ b/api_modules/mlxtend.classifier/StackingCVClassifier/index.html
@@ -12,25 +12,22 @@
         <link href="../../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../../cinder/css/base.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -986,14 +983,16 @@ <h3 id="properties">Properties</h3>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../../mathjaxhelper.js" defer></script>
-        <script src="../../../search/main.js" defer></script>
+        <script src="../../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../../mathjaxhelper.js"></script>
+        <script src="../../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/api_modules/mlxtend.classifier/StackingClassifier/index.html b/api_modules/mlxtend.classifier/StackingClassifier/index.html
index dd24fc128..38e6c9bb9 100644
--- a/api_modules/mlxtend.classifier/StackingClassifier/index.html
+++ b/api_modules/mlxtend.classifier/StackingClassifier/index.html
@@ -12,25 +12,22 @@
         <link href="../../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../../cinder/css/base.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -942,14 +939,16 @@ <h3 id="properties">Properties</h3>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../../mathjaxhelper.js" defer></script>
-        <script src="../../../search/main.js" defer></script>
+        <script src="../../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../../mathjaxhelper.js"></script>
+        <script src="../../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/api_modules/mlxtend.cluster/Kmeans/index.html b/api_modules/mlxtend.cluster/Kmeans/index.html
index 5cd9a0e44..4933ebbe0 100644
--- a/api_modules/mlxtend.cluster/Kmeans/index.html
+++ b/api_modules/mlxtend.cluster/Kmeans/index.html
@@ -12,25 +12,22 @@
         <link href="../../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../../cinder/css/base.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -764,14 +761,16 @@ <h3 id="methods">Methods</h3>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../../mathjaxhelper.js" defer></script>
-        <script src="../../../search/main.js" defer></script>
+        <script src="../../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../../mathjaxhelper.js"></script>
+        <script src="../../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/api_modules/mlxtend.data/autompg_data/index.html b/api_modules/mlxtend.data/autompg_data/index.html
index c0a68dbf7..408876d70 100644
--- a/api_modules/mlxtend.data/autompg_data/index.html
+++ b/api_modules/mlxtend.data/autompg_data/index.html
@@ -12,25 +12,22 @@
         <link href="../../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../../cinder/css/base.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -664,14 +661,16 @@ <h2 id="autompg_data">autompg_data</h2>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../../mathjaxhelper.js" defer></script>
-        <script src="../../../search/main.js" defer></script>
+        <script src="../../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../../mathjaxhelper.js"></script>
+        <script src="../../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/api_modules/mlxtend.data/boston_housing_data/index.html b/api_modules/mlxtend.data/boston_housing_data/index.html
index 29aa34bed..59344ffdd 100644
--- a/api_modules/mlxtend.data/boston_housing_data/index.html
+++ b/api_modules/mlxtend.data/boston_housing_data/index.html
@@ -12,25 +12,22 @@
         <link href="../../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../../cinder/css/base.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -672,14 +669,16 @@ <h2 id="boston_housing_data">boston_housing_data</h2>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../../mathjaxhelper.js" defer></script>
-        <script src="../../../search/main.js" defer></script>
+        <script src="../../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../../mathjaxhelper.js"></script>
+        <script src="../../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/api_modules/mlxtend.data/iris_data/index.html b/api_modules/mlxtend.data/iris_data/index.html
index a8eb11e65..2b425114f 100644
--- a/api_modules/mlxtend.data/iris_data/index.html
+++ b/api_modules/mlxtend.data/iris_data/index.html
@@ -12,25 +12,22 @@
         <link href="../../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../../cinder/css/base.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -684,14 +681,16 @@ <h2 id="iris_data">iris_data</h2>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../../mathjaxhelper.js" defer></script>
-        <script src="../../../search/main.js" defer></script>
+        <script src="../../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../../mathjaxhelper.js"></script>
+        <script src="../../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/api_modules/mlxtend.data/loadlocal_mnist/index.html b/api_modules/mlxtend.data/loadlocal_mnist/index.html
index ff7730435..a95cd0201 100644
--- a/api_modules/mlxtend.data/loadlocal_mnist/index.html
+++ b/api_modules/mlxtend.data/loadlocal_mnist/index.html
@@ -12,25 +12,22 @@
         <link href="../../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../../cinder/css/base.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -655,14 +652,16 @@ <h2 id="loadlocal_mnist">loadlocal_mnist</h2>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../../mathjaxhelper.js" defer></script>
-        <script src="../../../search/main.js" defer></script>
+        <script src="../../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../../mathjaxhelper.js"></script>
+        <script src="../../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/api_modules/mlxtend.data/make_multiplexer_dataset/index.html b/api_modules/mlxtend.data/make_multiplexer_dataset/index.html
index 2f0ce0698..aa754b273 100644
--- a/api_modules/mlxtend.data/make_multiplexer_dataset/index.html
+++ b/api_modules/mlxtend.data/make_multiplexer_dataset/index.html
@@ -12,25 +12,22 @@
         <link href="../../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../../cinder/css/base.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -686,14 +683,16 @@ <h2 id="make_multiplexer_dataset">make_multiplexer_dataset</h2>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../../mathjaxhelper.js" defer></script>
-        <script src="../../../search/main.js" defer></script>
+        <script src="../../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../../mathjaxhelper.js"></script>
+        <script src="../../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/api_modules/mlxtend.data/mnist_data/index.html b/api_modules/mlxtend.data/mnist_data/index.html
index 271cff7de..8cab2f6ca 100644
--- a/api_modules/mlxtend.data/mnist_data/index.html
+++ b/api_modules/mlxtend.data/mnist_data/index.html
@@ -12,25 +12,22 @@
         <link href="../../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../../cinder/css/base.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -646,14 +643,16 @@ <h2 id="mnist_data">mnist_data</h2>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../../mathjaxhelper.js" defer></script>
-        <script src="../../../search/main.js" defer></script>
+        <script src="../../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../../mathjaxhelper.js"></script>
+        <script src="../../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/api_modules/mlxtend.data/three_blobs_data/index.html b/api_modules/mlxtend.data/three_blobs_data/index.html
index 592e055c1..a578076e3 100644
--- a/api_modules/mlxtend.data/three_blobs_data/index.html
+++ b/api_modules/mlxtend.data/three_blobs_data/index.html
@@ -12,25 +12,22 @@
         <link href="../../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../../cinder/css/base.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -650,14 +647,16 @@ <h2 id="three_blobs_data">three_blobs_data</h2>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../../mathjaxhelper.js" defer></script>
-        <script src="../../../search/main.js" defer></script>
+        <script src="../../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../../mathjaxhelper.js"></script>
+        <script src="../../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/api_modules/mlxtend.data/wine_data/index.html b/api_modules/mlxtend.data/wine_data/index.html
index b6437f839..d57edaa84 100644
--- a/api_modules/mlxtend.data/wine_data/index.html
+++ b/api_modules/mlxtend.data/wine_data/index.html
@@ -12,25 +12,22 @@
         <link href="../../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../../cinder/css/base.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -669,14 +666,16 @@ <h2 id="wine_data">wine_data</h2>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../../mathjaxhelper.js" defer></script>
-        <script src="../../../search/main.js" defer></script>
+        <script src="../../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../../mathjaxhelper.js"></script>
+        <script src="../../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/api_modules/mlxtend.evaluate/BootstrapOutOfBag/index.html b/api_modules/mlxtend.evaluate/BootstrapOutOfBag/index.html
index c5ea37f7c..58101477a 100644
--- a/api_modules/mlxtend.evaluate/BootstrapOutOfBag/index.html
+++ b/api_modules/mlxtend.evaluate/BootstrapOutOfBag/index.html
@@ -12,25 +12,22 @@
         <link href="../../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../../cinder/css/base.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -696,14 +693,16 @@ <h3 id="methods">Methods</h3>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../../mathjaxhelper.js" defer></script>
-        <script src="../../../search/main.js" defer></script>
+        <script src="../../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../../mathjaxhelper.js"></script>
+        <script src="../../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/api_modules/mlxtend.evaluate/GroupTimeSeriesSplit/index.html b/api_modules/mlxtend.evaluate/GroupTimeSeriesSplit/index.html
index 5772593c9..0db80712a 100644
--- a/api_modules/mlxtend.evaluate/GroupTimeSeriesSplit/index.html
+++ b/api_modules/mlxtend.evaluate/GroupTimeSeriesSplit/index.html
@@ -12,25 +12,22 @@
         <link href="../../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../../cinder/css/base.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -717,14 +714,16 @@ <h3 id="methods">Methods</h3>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../../mathjaxhelper.js" defer></script>
-        <script src="../../../search/main.js" defer></script>
+        <script src="../../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../../mathjaxhelper.js"></script>
+        <script src="../../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/api_modules/mlxtend.evaluate/PredefinedHoldoutSplit/index.html b/api_modules/mlxtend.evaluate/PredefinedHoldoutSplit/index.html
index 8dba8c977..ff032747e 100644
--- a/api_modules/mlxtend.evaluate/PredefinedHoldoutSplit/index.html
+++ b/api_modules/mlxtend.evaluate/PredefinedHoldoutSplit/index.html
@@ -12,25 +12,22 @@
         <link href="../../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../../cinder/css/base.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -707,14 +704,16 @@ <h3 id="methods">Methods</h3>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../../mathjaxhelper.js" defer></script>
-        <script src="../../../search/main.js" defer></script>
+        <script src="../../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../../mathjaxhelper.js"></script>
+        <script src="../../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/api_modules/mlxtend.evaluate/RandomHoldoutSplit/index.html b/api_modules/mlxtend.evaluate/RandomHoldoutSplit/index.html
index 48947704e..8eeab3a0c 100644
--- a/api_modules/mlxtend.evaluate/RandomHoldoutSplit/index.html
+++ b/api_modules/mlxtend.evaluate/RandomHoldoutSplit/index.html
@@ -12,25 +12,22 @@
         <link href="../../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../../cinder/css/base.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -715,14 +712,16 @@ <h3 id="methods">Methods</h3>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../../mathjaxhelper.js" defer></script>
-        <script src="../../../search/main.js" defer></script>
+        <script src="../../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../../mathjaxhelper.js"></script>
+        <script src="../../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/api_modules/mlxtend.evaluate/accuracy_score/index.html b/api_modules/mlxtend.evaluate/accuracy_score/index.html
index 160baa36a..9a97bb46b 100644
--- a/api_modules/mlxtend.evaluate/accuracy_score/index.html
+++ b/api_modules/mlxtend.evaluate/accuracy_score/index.html
@@ -12,25 +12,22 @@
         <link href="../../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../../cinder/css/base.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -664,14 +661,16 @@ <h2 id="accuracy_score">accuracy_score</h2>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../../mathjaxhelper.js" defer></script>
-        <script src="../../../search/main.js" defer></script>
+        <script src="../../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../../mathjaxhelper.js"></script>
+        <script src="../../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/api_modules/mlxtend.evaluate/bias_variance_decomp/index.html b/api_modules/mlxtend.evaluate/bias_variance_decomp/index.html
index 0aec56e2a..be23198c6 100644
--- a/api_modules/mlxtend.evaluate/bias_variance_decomp/index.html
+++ b/api_modules/mlxtend.evaluate/bias_variance_decomp/index.html
@@ -12,25 +12,22 @@
         <link href="../../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../../cinder/css/base.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -686,14 +683,16 @@ <h2 id="bias_variance_decomp">bias_variance_decomp</h2>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../../mathjaxhelper.js" defer></script>
-        <script src="../../../search/main.js" defer></script>
+        <script src="../../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../../mathjaxhelper.js"></script>
+        <script src="../../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/api_modules/mlxtend.evaluate/bootstrap/index.html b/api_modules/mlxtend.evaluate/bootstrap/index.html
index 7d544f8bb..6144e84f5 100644
--- a/api_modules/mlxtend.evaluate/bootstrap/index.html
+++ b/api_modules/mlxtend.evaluate/bootstrap/index.html
@@ -12,25 +12,22 @@
         <link href="../../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../../cinder/css/base.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -699,14 +696,16 @@ <h2 id="bootstrap">bootstrap</h2>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../../mathjaxhelper.js" defer></script>
-        <script src="../../../search/main.js" defer></script>
+        <script src="../../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../../mathjaxhelper.js"></script>
+        <script src="../../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/api_modules/mlxtend.evaluate/bootstrap_point632_score/index.html b/api_modules/mlxtend.evaluate/bootstrap_point632_score/index.html
index 245ef7a84..6961c198e 100644
--- a/api_modules/mlxtend.evaluate/bootstrap_point632_score/index.html
+++ b/api_modules/mlxtend.evaluate/bootstrap_point632_score/index.html
@@ -12,25 +12,22 @@
         <link href="../../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../../cinder/css/base.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -737,14 +734,16 @@ <h2 id="bootstrap_point632_score">bootstrap_point632_score</h2>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../../mathjaxhelper.js" defer></script>
-        <script src="../../../search/main.js" defer></script>
+        <script src="../../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../../mathjaxhelper.js"></script>
+        <script src="../../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/api_modules/mlxtend.evaluate/cochrans_q/index.html b/api_modules/mlxtend.evaluate/cochrans_q/index.html
index fa4134683..2c15632d3 100644
--- a/api_modules/mlxtend.evaluate/cochrans_q/index.html
+++ b/api_modules/mlxtend.evaluate/cochrans_q/index.html
@@ -12,25 +12,22 @@
         <link href="../../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../../cinder/css/base.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -653,14 +650,16 @@ <h2 id="cochrans_q">cochrans_q</h2>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../../mathjaxhelper.js" defer></script>
-        <script src="../../../search/main.js" defer></script>
+        <script src="../../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../../mathjaxhelper.js"></script>
+        <script src="../../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/api_modules/mlxtend.evaluate/combined_ftest_5x2cv/index.html b/api_modules/mlxtend.evaluate/combined_ftest_5x2cv/index.html
index 54c3656a1..4101b4878 100644
--- a/api_modules/mlxtend.evaluate/combined_ftest_5x2cv/index.html
+++ b/api_modules/mlxtend.evaluate/combined_ftest_5x2cv/index.html
@@ -12,25 +12,22 @@
         <link href="../../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../../cinder/css/base.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -685,14 +682,16 @@ <h2 id="combined_ftest_5x2cv">combined_ftest_5x2cv</h2>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../../mathjaxhelper.js" defer></script>
-        <script src="../../../search/main.js" defer></script>
+        <script src="../../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../../mathjaxhelper.js"></script>
+        <script src="../../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/api_modules/mlxtend.evaluate/confusion_matrix/index.html b/api_modules/mlxtend.evaluate/confusion_matrix/index.html
index e24d285a9..6228e4fbd 100644
--- a/api_modules/mlxtend.evaluate/confusion_matrix/index.html
+++ b/api_modules/mlxtend.evaluate/confusion_matrix/index.html
@@ -12,25 +12,22 @@
         <link href="../../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../../cinder/css/base.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -659,14 +656,16 @@ <h2 id="confusion_matrix">confusion_matrix</h2>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../../mathjaxhelper.js" defer></script>
-        <script src="../../../search/main.js" defer></script>
+        <script src="../../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../../mathjaxhelper.js"></script>
+        <script src="../../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/api_modules/mlxtend.evaluate/create_counterfactual/index.html b/api_modules/mlxtend.evaluate/create_counterfactual/index.html
index 9e5a3b2b0..77105383d 100644
--- a/api_modules/mlxtend.evaluate/create_counterfactual/index.html
+++ b/api_modules/mlxtend.evaluate/create_counterfactual/index.html
@@ -12,25 +12,22 @@
         <link href="../../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../../cinder/css/base.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -682,14 +679,16 @@ <h2 id="create_counterfactual">create_counterfactual</h2>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../../mathjaxhelper.js" defer></script>
-        <script src="../../../search/main.js" defer></script>
+        <script src="../../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../../mathjaxhelper.js"></script>
+        <script src="../../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/api_modules/mlxtend.evaluate/feature_importance_permutation/index.html b/api_modules/mlxtend.evaluate/feature_importance_permutation/index.html
index 231b8fc9f..b4d78cde3 100644
--- a/api_modules/mlxtend.evaluate/feature_importance_permutation/index.html
+++ b/api_modules/mlxtend.evaluate/feature_importance_permutation/index.html
@@ -12,25 +12,22 @@
         <link href="../../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../../cinder/css/base.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -687,14 +684,16 @@ <h2 id="feature_importance_permutation">feature_importance_permutation</h2>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../../mathjaxhelper.js" defer></script>
-        <script src="../../../search/main.js" defer></script>
+        <script src="../../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../../mathjaxhelper.js"></script>
+        <script src="../../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/api_modules/mlxtend.evaluate/ftest/index.html b/api_modules/mlxtend.evaluate/ftest/index.html
index 984015ec6..56caaabf0 100644
--- a/api_modules/mlxtend.evaluate/ftest/index.html
+++ b/api_modules/mlxtend.evaluate/ftest/index.html
@@ -12,25 +12,22 @@
         <link href="../../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../../cinder/css/base.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -653,14 +650,16 @@ <h2 id="ftest">ftest</h2>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../../mathjaxhelper.js" defer></script>
-        <script src="../../../search/main.js" defer></script>
+        <script src="../../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../../mathjaxhelper.js"></script>
+        <script src="../../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/api_modules/mlxtend.evaluate/lift_score/index.html b/api_modules/mlxtend.evaluate/lift_score/index.html
index c57938f5d..5eabc7b6f 100644
--- a/api_modules/mlxtend.evaluate/lift_score/index.html
+++ b/api_modules/mlxtend.evaluate/lift_score/index.html
@@ -12,25 +12,22 @@
         <link href="../../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../../cinder/css/base.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -668,14 +665,16 @@ <h2 id="lift_score">lift_score</h2>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../../mathjaxhelper.js" defer></script>
-        <script src="../../../search/main.js" defer></script>
+        <script src="../../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../../mathjaxhelper.js"></script>
+        <script src="../../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/api_modules/mlxtend.evaluate/mcnemar/index.html b/api_modules/mlxtend.evaluate/mcnemar/index.html
index f9f4a8c8f..a8c0089ed 100644
--- a/api_modules/mlxtend.evaluate/mcnemar/index.html
+++ b/api_modules/mlxtend.evaluate/mcnemar/index.html
@@ -12,25 +12,22 @@
         <link href="../../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../../cinder/css/base.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -666,14 +663,16 @@ <h2 id="mcnemar">mcnemar</h2>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../../mathjaxhelper.js" defer></script>
-        <script src="../../../search/main.js" defer></script>
+        <script src="../../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../../mathjaxhelper.js"></script>
+        <script src="../../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/api_modules/mlxtend.evaluate/mcnemar_table/index.html b/api_modules/mlxtend.evaluate/mcnemar_table/index.html
index 37f825528..ba6d6708f 100644
--- a/api_modules/mlxtend.evaluate/mcnemar_table/index.html
+++ b/api_modules/mlxtend.evaluate/mcnemar_table/index.html
@@ -12,25 +12,22 @@
         <link href="../../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../../cinder/css/base.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -659,14 +656,16 @@ <h2 id="mcnemar_table">mcnemar_table</h2>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../../mathjaxhelper.js" defer></script>
-        <script src="../../../search/main.js" defer></script>
+        <script src="../../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../../mathjaxhelper.js"></script>
+        <script src="../../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/api_modules/mlxtend.evaluate/mcnemar_tables/index.html b/api_modules/mlxtend.evaluate/mcnemar_tables/index.html
index f03828752..9378e2d71 100644
--- a/api_modules/mlxtend.evaluate/mcnemar_tables/index.html
+++ b/api_modules/mlxtend.evaluate/mcnemar_tables/index.html
@@ -12,25 +12,22 @@
         <link href="../../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../../cinder/css/base.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -679,14 +676,16 @@ <h2 id="mcnemar_tables">mcnemar_tables</h2>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../../mathjaxhelper.js" defer></script>
-        <script src="../../../search/main.js" defer></script>
+        <script src="../../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../../mathjaxhelper.js"></script>
+        <script src="../../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/api_modules/mlxtend.evaluate/paired_ttest_5x2cv/index.html b/api_modules/mlxtend.evaluate/paired_ttest_5x2cv/index.html
index 829ee9c34..0e34d62e6 100644
--- a/api_modules/mlxtend.evaluate/paired_ttest_5x2cv/index.html
+++ b/api_modules/mlxtend.evaluate/paired_ttest_5x2cv/index.html
@@ -12,25 +12,22 @@
         <link href="../../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../../cinder/css/base.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -685,14 +682,16 @@ <h2 id="paired_ttest_5x2cv">paired_ttest_5x2cv</h2>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../../mathjaxhelper.js" defer></script>
-        <script src="../../../search/main.js" defer></script>
+        <script src="../../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../../mathjaxhelper.js"></script>
+        <script src="../../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/api_modules/mlxtend.evaluate/paired_ttest_kfold_cv/index.html b/api_modules/mlxtend.evaluate/paired_ttest_kfold_cv/index.html
index 2bbd0455b..e323c260f 100644
--- a/api_modules/mlxtend.evaluate/paired_ttest_kfold_cv/index.html
+++ b/api_modules/mlxtend.evaluate/paired_ttest_kfold_cv/index.html
@@ -12,25 +12,22 @@
         <link href="../../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../../cinder/css/base.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -696,14 +693,16 @@ <h2 id="paired_ttest_kfold_cv">paired_ttest_kfold_cv</h2>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../../mathjaxhelper.js" defer></script>
-        <script src="../../../search/main.js" defer></script>
+        <script src="../../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../../mathjaxhelper.js"></script>
+        <script src="../../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/api_modules/mlxtend.evaluate/paired_ttest_resampled/index.html b/api_modules/mlxtend.evaluate/paired_ttest_resampled/index.html
index 4ce74a539..b96518a62 100644
--- a/api_modules/mlxtend.evaluate/paired_ttest_resampled/index.html
+++ b/api_modules/mlxtend.evaluate/paired_ttest_resampled/index.html
@@ -12,25 +12,22 @@
         <link href="../../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../../cinder/css/base.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -697,14 +694,16 @@ <h2 id="paired_ttest_resampled">paired_ttest_resampled</h2>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../../mathjaxhelper.js" defer></script>
-        <script src="../../../search/main.js" defer></script>
+        <script src="../../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../../mathjaxhelper.js"></script>
+        <script src="../../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/api_modules/mlxtend.evaluate/permutation_test/index.html b/api_modules/mlxtend.evaluate/permutation_test/index.html
index 0b08c0eba..71f86a64a 100644
--- a/api_modules/mlxtend.evaluate/permutation_test/index.html
+++ b/api_modules/mlxtend.evaluate/permutation_test/index.html
@@ -12,25 +12,22 @@
         <link href="../../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../../cinder/css/base.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -684,14 +681,16 @@ <h2 id="permutation_test">permutation_test</h2>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../../mathjaxhelper.js" defer></script>
-        <script src="../../../search/main.js" defer></script>
+        <script src="../../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../../mathjaxhelper.js"></script>
+        <script src="../../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/api_modules/mlxtend.evaluate/proportion_difference/index.html b/api_modules/mlxtend.evaluate/proportion_difference/index.html
index de8a5c90b..d2aaee0bd 100644
--- a/api_modules/mlxtend.evaluate/proportion_difference/index.html
+++ b/api_modules/mlxtend.evaluate/proportion_difference/index.html
@@ -12,25 +12,22 @@
         <link href="../../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../../cinder/css/base.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -661,14 +658,16 @@ <h2 id="proportion_difference">proportion_difference</h2>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../../mathjaxhelper.js" defer></script>
-        <script src="../../../search/main.js" defer></script>
+        <script src="../../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../../mathjaxhelper.js"></script>
+        <script src="../../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/api_modules/mlxtend.evaluate/scoring/index.html b/api_modules/mlxtend.evaluate/scoring/index.html
index 68c101ea8..abe21f28c 100644
--- a/api_modules/mlxtend.evaluate/scoring/index.html
+++ b/api_modules/mlxtend.evaluate/scoring/index.html
@@ -12,25 +12,22 @@
         <link href="../../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../../cinder/css/base.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -681,14 +678,16 @@ <h2 id="scoring">scoring</h2>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../../mathjaxhelper.js" defer></script>
-        <script src="../../../search/main.js" defer></script>
+        <script src="../../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../../mathjaxhelper.js"></script>
+        <script src="../../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/api_modules/mlxtend.feature_extraction/LinearDiscriminantAnalysis/index.html b/api_modules/mlxtend.feature_extraction/LinearDiscriminantAnalysis/index.html
index 29973d905..12fc219c8 100644
--- a/api_modules/mlxtend.feature_extraction/LinearDiscriminantAnalysis/index.html
+++ b/api_modules/mlxtend.feature_extraction/LinearDiscriminantAnalysis/index.html
@@ -12,25 +12,22 @@
         <link href="../../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../../cinder/css/base.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -740,14 +737,16 @@ <h3 id="methods">Methods</h3>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../../mathjaxhelper.js" defer></script>
-        <script src="../../../search/main.js" defer></script>
+        <script src="../../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../../mathjaxhelper.js"></script>
+        <script src="../../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/api_modules/mlxtend.feature_extraction/PrincipalComponentAnalysis/index.html b/api_modules/mlxtend.feature_extraction/PrincipalComponentAnalysis/index.html
index 0540d6757..a60a6efa6 100644
--- a/api_modules/mlxtend.feature_extraction/PrincipalComponentAnalysis/index.html
+++ b/api_modules/mlxtend.feature_extraction/PrincipalComponentAnalysis/index.html
@@ -12,25 +12,22 @@
         <link href="../../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../../cinder/css/base.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -757,14 +754,16 @@ <h3 id="methods">Methods</h3>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../../mathjaxhelper.js" defer></script>
-        <script src="../../../search/main.js" defer></script>
+        <script src="../../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../../mathjaxhelper.js"></script>
+        <script src="../../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/api_modules/mlxtend.feature_extraction/RBFKernelPCA/index.html b/api_modules/mlxtend.feature_extraction/RBFKernelPCA/index.html
index d5603141c..748bb4e3e 100644
--- a/api_modules/mlxtend.feature_extraction/RBFKernelPCA/index.html
+++ b/api_modules/mlxtend.feature_extraction/RBFKernelPCA/index.html
@@ -12,25 +12,22 @@
         <link href="../../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../../cinder/css/base.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -739,14 +736,16 @@ <h3 id="methods">Methods</h3>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../../mathjaxhelper.js" defer></script>
-        <script src="../../../search/main.js" defer></script>
+        <script src="../../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../../mathjaxhelper.js"></script>
+        <script src="../../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/api_modules/mlxtend.feature_selection/ColumnSelector/index.html b/api_modules/mlxtend.feature_selection/ColumnSelector/index.html
index b5adf89ba..db8e57b1a 100644
--- a/api_modules/mlxtend.feature_selection/ColumnSelector/index.html
+++ b/api_modules/mlxtend.feature_selection/ColumnSelector/index.html
@@ -12,25 +12,22 @@
         <link href="../../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../../cinder/css/base.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -756,14 +753,16 @@ <h3 id="methods">Methods</h3>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../../mathjaxhelper.js" defer></script>
-        <script src="../../../search/main.js" defer></script>
+        <script src="../../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../../mathjaxhelper.js"></script>
+        <script src="../../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/api_modules/mlxtend.feature_selection/ExhaustiveFeatureSelector/index.html b/api_modules/mlxtend.feature_selection/ExhaustiveFeatureSelector/index.html
index c4810f9db..48049cd09 100644
--- a/api_modules/mlxtend.feature_selection/ExhaustiveFeatureSelector/index.html
+++ b/api_modules/mlxtend.feature_selection/ExhaustiveFeatureSelector/index.html
@@ -12,25 +12,22 @@
         <link href="../../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../../cinder/css/base.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -924,14 +921,16 @@ <h3 id="methods">Methods</h3>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../../mathjaxhelper.js" defer></script>
-        <script src="../../../search/main.js" defer></script>
+        <script src="../../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../../mathjaxhelper.js"></script>
+        <script src="../../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/api_modules/mlxtend.feature_selection/SequentialFeatureSelector/index.html b/api_modules/mlxtend.feature_selection/SequentialFeatureSelector/index.html
index 4f351b9ab..285652ef3 100644
--- a/api_modules/mlxtend.feature_selection/SequentialFeatureSelector/index.html
+++ b/api_modules/mlxtend.feature_selection/SequentialFeatureSelector/index.html
@@ -12,25 +12,22 @@
         <link href="../../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../../cinder/css/base.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -944,14 +941,16 @@ <h3 id="properties">Properties</h3>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../../mathjaxhelper.js" defer></script>
-        <script src="../../../search/main.js" defer></script>
+        <script src="../../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../../mathjaxhelper.js"></script>
+        <script src="../../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/api_modules/mlxtend.file_io/find_filegroups/index.html b/api_modules/mlxtend.file_io/find_filegroups/index.html
index bd5bfd5cb..34a0bf08d 100644
--- a/api_modules/mlxtend.file_io/find_filegroups/index.html
+++ b/api_modules/mlxtend.file_io/find_filegroups/index.html
@@ -12,25 +12,22 @@
         <link href="../../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../../cinder/css/base.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -682,14 +679,16 @@ <h2 id="find_filegroups">find_filegroups</h2>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../../mathjaxhelper.js" defer></script>
-        <script src="../../../search/main.js" defer></script>
+        <script src="../../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../../mathjaxhelper.js"></script>
+        <script src="../../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/api_modules/mlxtend.file_io/find_files/index.html b/api_modules/mlxtend.file_io/find_files/index.html
index 13ac5287a..22af4e619 100644
--- a/api_modules/mlxtend.file_io/find_files/index.html
+++ b/api_modules/mlxtend.file_io/find_files/index.html
@@ -12,25 +12,22 @@
         <link href="../../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../../cinder/css/base.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -669,14 +666,16 @@ <h2 id="find_files">find_files</h2>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../../mathjaxhelper.js" defer></script>
-        <script src="../../../search/main.js" defer></script>
+        <script src="../../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../../mathjaxhelper.js"></script>
+        <script src="../../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/api_modules/mlxtend.frequent_patterns/apriori/index.html b/api_modules/mlxtend.frequent_patterns/apriori/index.html
index 209e36a0c..21f1f1b63 100644
--- a/api_modules/mlxtend.frequent_patterns/apriori/index.html
+++ b/api_modules/mlxtend.frequent_patterns/apriori/index.html
@@ -12,25 +12,22 @@
         <link href="../../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../../cinder/css/base.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -696,14 +693,16 @@ <h2 id="apriori">apriori</h2>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../../mathjaxhelper.js" defer></script>
-        <script src="../../../search/main.js" defer></script>
+        <script src="../../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../../mathjaxhelper.js"></script>
+        <script src="../../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/api_modules/mlxtend.frequent_patterns/association_rules/index.html b/api_modules/mlxtend.frequent_patterns/association_rules/index.html
index 24ae46a15..c17de65e4 100644
--- a/api_modules/mlxtend.frequent_patterns/association_rules/index.html
+++ b/api_modules/mlxtend.frequent_patterns/association_rules/index.html
@@ -12,25 +12,22 @@
         <link href="../../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../../cinder/css/base.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -697,14 +694,16 @@ <h2 id="association_rules">association_rules</h2>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../../mathjaxhelper.js" defer></script>
-        <script src="../../../search/main.js" defer></script>
+        <script src="../../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../../mathjaxhelper.js"></script>
+        <script src="../../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/api_modules/mlxtend.frequent_patterns/fpgrowth/index.html b/api_modules/mlxtend.frequent_patterns/fpgrowth/index.html
index 8ccbb26b6..7c466c12e 100644
--- a/api_modules/mlxtend.frequent_patterns/fpgrowth/index.html
+++ b/api_modules/mlxtend.frequent_patterns/fpgrowth/index.html
@@ -12,25 +12,22 @@
         <link href="../../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../../cinder/css/base.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -684,14 +681,16 @@ <h2 id="fpgrowth">fpgrowth</h2>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../../mathjaxhelper.js" defer></script>
-        <script src="../../../search/main.js" defer></script>
+        <script src="../../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../../mathjaxhelper.js"></script>
+        <script src="../../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/api_modules/mlxtend.frequent_patterns/fpmax/index.html b/api_modules/mlxtend.frequent_patterns/fpmax/index.html
index 8bb016287..661186227 100644
--- a/api_modules/mlxtend.frequent_patterns/fpmax/index.html
+++ b/api_modules/mlxtend.frequent_patterns/fpmax/index.html
@@ -12,25 +12,22 @@
         <link href="../../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../../cinder/css/base.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -686,14 +683,16 @@ <h2 id="fpmax">fpmax</h2>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../../mathjaxhelper.js" defer></script>
-        <script src="../../../search/main.js" defer></script>
+        <script src="../../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../../mathjaxhelper.js"></script>
+        <script src="../../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/api_modules/mlxtend.frequent_patterns/hmine/index.html b/api_modules/mlxtend.frequent_patterns/hmine/index.html
index f053cfec4..efcee3e14 100644
--- a/api_modules/mlxtend.frequent_patterns/hmine/index.html
+++ b/api_modules/mlxtend.frequent_patterns/hmine/index.html
@@ -12,25 +12,22 @@
         <link href="../../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../../cinder/css/base.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -684,14 +681,16 @@ <h2 id="hmine">hmine</h2>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../../mathjaxhelper.js" defer></script>
-        <script src="../../../search/main.js" defer></script>
+        <script src="../../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../../mathjaxhelper.js"></script>
+        <script src="../../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/api_modules/mlxtend.image/EyepadAlign/index.html b/api_modules/mlxtend.image/EyepadAlign/index.html
index 8fec92f25..f7dbacbee 100644
--- a/api_modules/mlxtend.image/EyepadAlign/index.html
+++ b/api_modules/mlxtend.image/EyepadAlign/index.html
@@ -12,25 +12,22 @@
         <link href="../../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../../cinder/css/base.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -775,14 +772,16 @@ <h3 id="methods">Methods</h3>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../../mathjaxhelper.js" defer></script>
-        <script src="../../../search/main.js" defer></script>
+        <script src="../../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../../mathjaxhelper.js"></script>
+        <script src="../../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/api_modules/mlxtend.image/extract_face_landmarks/index.html b/api_modules/mlxtend.image/extract_face_landmarks/index.html
index 4df893431..49701c4a9 100644
--- a/api_modules/mlxtend.image/extract_face_landmarks/index.html
+++ b/api_modules/mlxtend.image/extract_face_landmarks/index.html
@@ -12,25 +12,22 @@
         <link href="../../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../../cinder/css/base.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -661,14 +658,16 @@ <h2 id="extract_face_landmarks">extract_face_landmarks</h2>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../../mathjaxhelper.js" defer></script>
-        <script src="../../../search/main.js" defer></script>
+        <script src="../../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../../mathjaxhelper.js"></script>
+        <script src="../../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/api_modules/mlxtend.math/factorial/index.html b/api_modules/mlxtend.math/factorial/index.html
index 441d38b08..fa913a429 100644
--- a/api_modules/mlxtend.math/factorial/index.html
+++ b/api_modules/mlxtend.math/factorial/index.html
@@ -12,25 +12,22 @@
         <link href="../../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../../cinder/css/base.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -630,14 +627,16 @@ <h2 id="factorial">factorial</h2>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../../mathjaxhelper.js" defer></script>
-        <script src="../../../search/main.js" defer></script>
+        <script src="../../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../../mathjaxhelper.js"></script>
+        <script src="../../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/api_modules/mlxtend.math/num_combinations/index.html b/api_modules/mlxtend.math/num_combinations/index.html
index efd72e4dd..76f6508ad 100644
--- a/api_modules/mlxtend.math/num_combinations/index.html
+++ b/api_modules/mlxtend.math/num_combinations/index.html
@@ -12,25 +12,22 @@
         <link href="../../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../../cinder/css/base.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -655,14 +652,16 @@ <h2 id="num_combinations">num_combinations</h2>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../../mathjaxhelper.js" defer></script>
-        <script src="../../../search/main.js" defer></script>
+        <script src="../../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../../mathjaxhelper.js"></script>
+        <script src="../../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/api_modules/mlxtend.math/num_permutations/index.html b/api_modules/mlxtend.math/num_permutations/index.html
index 21707fb64..88f78e9de 100644
--- a/api_modules/mlxtend.math/num_permutations/index.html
+++ b/api_modules/mlxtend.math/num_permutations/index.html
@@ -12,25 +12,22 @@
         <link href="../../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../../cinder/css/base.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -655,14 +652,16 @@ <h2 id="num_permutations">num_permutations</h2>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../../mathjaxhelper.js" defer></script>
-        <script src="../../../search/main.js" defer></script>
+        <script src="../../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../../mathjaxhelper.js"></script>
+        <script src="../../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/api_modules/mlxtend.math/vectorspace_dimensionality/index.html b/api_modules/mlxtend.math/vectorspace_dimensionality/index.html
index 55832ea6e..8cc0b14d6 100644
--- a/api_modules/mlxtend.math/vectorspace_dimensionality/index.html
+++ b/api_modules/mlxtend.math/vectorspace_dimensionality/index.html
@@ -12,25 +12,22 @@
         <link href="../../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../../cinder/css/base.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -648,14 +645,16 @@ <h2 id="vectorspace_dimensionality">vectorspace_dimensionality</h2>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../../mathjaxhelper.js" defer></script>
-        <script src="../../../search/main.js" defer></script>
+        <script src="../../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../../mathjaxhelper.js"></script>
+        <script src="../../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/api_modules/mlxtend.math/vectorspace_orthonormalization/index.html b/api_modules/mlxtend.math/vectorspace_orthonormalization/index.html
index 204eba389..3d106cbc0 100644
--- a/api_modules/mlxtend.math/vectorspace_orthonormalization/index.html
+++ b/api_modules/mlxtend.math/vectorspace_orthonormalization/index.html
@@ -12,25 +12,22 @@
         <link href="../../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../../cinder/css/base.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -656,14 +653,16 @@ <h2 id="vectorspace_orthonormalization">vectorspace_orthonormalization</h2>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../../mathjaxhelper.js" defer></script>
-        <script src="../../../search/main.js" defer></script>
+        <script src="../../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../../mathjaxhelper.js"></script>
+        <script src="../../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/api_modules/mlxtend.plotting/category_scatter/index.html b/api_modules/mlxtend.plotting/category_scatter/index.html
index 06f165d37..30d6e841a 100644
--- a/api_modules/mlxtend.plotting/category_scatter/index.html
+++ b/api_modules/mlxtend.plotting/category_scatter/index.html
@@ -12,25 +12,22 @@
         <link href="../../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../../cinder/css/base.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -675,14 +672,16 @@ <h2 id="category_scatter">category_scatter</h2>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../../mathjaxhelper.js" defer></script>
-        <script src="../../../search/main.js" defer></script>
+        <script src="../../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../../mathjaxhelper.js"></script>
+        <script src="../../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/api_modules/mlxtend.plotting/checkerboard_plot/index.html b/api_modules/mlxtend.plotting/checkerboard_plot/index.html
index f44b5d28e..b5d88122e 100644
--- a/api_modules/mlxtend.plotting/checkerboard_plot/index.html
+++ b/api_modules/mlxtend.plotting/checkerboard_plot/index.html
@@ -12,25 +12,22 @@
         <link href="../../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../../cinder/css/base.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -678,14 +675,16 @@ <h2 id="checkerboard_plot">checkerboard_plot</h2>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../../mathjaxhelper.js" defer></script>
-        <script src="../../../search/main.js" defer></script>
+        <script src="../../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../../mathjaxhelper.js"></script>
+        <script src="../../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/api_modules/mlxtend.plotting/ecdf/index.html b/api_modules/mlxtend.plotting/ecdf/index.html
index 7ebcf342d..4e9b3afa4 100644
--- a/api_modules/mlxtend.plotting/ecdf/index.html
+++ b/api_modules/mlxtend.plotting/ecdf/index.html
@@ -12,25 +12,22 @@
         <link href="../../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../../cinder/css/base.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -691,14 +688,16 @@ <h2 id="ecdf">ecdf</h2>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../../mathjaxhelper.js" defer></script>
-        <script src="../../../search/main.js" defer></script>
+        <script src="../../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../../mathjaxhelper.js"></script>
+        <script src="../../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/api_modules/mlxtend.plotting/enrichment_plot/index.html b/api_modules/mlxtend.plotting/enrichment_plot/index.html
index 7dd5ee584..1eff2eb4e 100644
--- a/api_modules/mlxtend.plotting/enrichment_plot/index.html
+++ b/api_modules/mlxtend.plotting/enrichment_plot/index.html
@@ -12,25 +12,22 @@
         <link href="../../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../../cinder/css/base.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -699,14 +696,16 @@ <h2 id="enrichment_plot">enrichment_plot</h2>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../../mathjaxhelper.js" defer></script>
-        <script src="../../../search/main.js" defer></script>
+        <script src="../../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../../mathjaxhelper.js"></script>
+        <script src="../../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/api_modules/mlxtend.plotting/heatmap/index.html b/api_modules/mlxtend.plotting/heatmap/index.html
index efb1e30b1..5706715a2 100644
--- a/api_modules/mlxtend.plotting/heatmap/index.html
+++ b/api_modules/mlxtend.plotting/heatmap/index.html
@@ -12,25 +12,22 @@
         <link href="../../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../../cinder/css/base.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -697,14 +694,16 @@ <h2 id="heatmap">heatmap</h2>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../../mathjaxhelper.js" defer></script>
-        <script src="../../../search/main.js" defer></script>
+        <script src="../../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../../mathjaxhelper.js"></script>
+        <script src="../../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/api_modules/mlxtend.plotting/plot_confusion_matrix/index.html b/api_modules/mlxtend.plotting/plot_confusion_matrix/index.html
index 30dd2f6ad..b0de66d11 100644
--- a/api_modules/mlxtend.plotting/plot_confusion_matrix/index.html
+++ b/api_modules/mlxtend.plotting/plot_confusion_matrix/index.html
@@ -12,25 +12,22 @@
         <link href="../../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../../cinder/css/base.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -708,14 +705,16 @@ <h2 id="plot_confusion_matrix">plot_confusion_matrix</h2>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../../mathjaxhelper.js" defer></script>
-        <script src="../../../search/main.js" defer></script>
+        <script src="../../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../../mathjaxhelper.js"></script>
+        <script src="../../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/api_modules/mlxtend.plotting/plot_decision_regions/index.html b/api_modules/mlxtend.plotting/plot_decision_regions/index.html
index 49eb2b6fe..958a187cf 100644
--- a/api_modules/mlxtend.plotting/plot_decision_regions/index.html
+++ b/api_modules/mlxtend.plotting/plot_decision_regions/index.html
@@ -12,25 +12,22 @@
         <link href="../../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../../cinder/css/base.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -730,14 +727,16 @@ <h2 id="plot_decision_regions">plot_decision_regions</h2>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../../mathjaxhelper.js" defer></script>
-        <script src="../../../search/main.js" defer></script>
+        <script src="../../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../../mathjaxhelper.js"></script>
+        <script src="../../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/api_modules/mlxtend.plotting/plot_learning_curves/index.html b/api_modules/mlxtend.plotting/plot_learning_curves/index.html
index 9ae93eb86..c32440f7b 100644
--- a/api_modules/mlxtend.plotting/plot_learning_curves/index.html
+++ b/api_modules/mlxtend.plotting/plot_learning_curves/index.html
@@ -12,25 +12,22 @@
         <link href="../../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../../cinder/css/base.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -700,14 +697,16 @@ <h2 id="plot_learning_curves">plot_learning_curves</h2>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../../mathjaxhelper.js" defer></script>
-        <script src="../../../search/main.js" defer></script>
+        <script src="../../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../../mathjaxhelper.js"></script>
+        <script src="../../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/api_modules/mlxtend.plotting/plot_linear_regression/index.html b/api_modules/mlxtend.plotting/plot_linear_regression/index.html
index 3f1210f5b..dfa79c58b 100644
--- a/api_modules/mlxtend.plotting/plot_linear_regression/index.html
+++ b/api_modules/mlxtend.plotting/plot_linear_regression/index.html
@@ -12,25 +12,22 @@
         <link href="../../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../../cinder/css/base.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -669,14 +666,16 @@ <h2 id="plot_linear_regression">plot_linear_regression</h2>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../../mathjaxhelper.js" defer></script>
-        <script src="../../../search/main.js" defer></script>
+        <script src="../../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../../mathjaxhelper.js"></script>
+        <script src="../../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/api_modules/mlxtend.plotting/plot_pca_correlation_graph/index.html b/api_modules/mlxtend.plotting/plot_pca_correlation_graph/index.html
index 1bb07d5d1..8431b71eb 100644
--- a/api_modules/mlxtend.plotting/plot_pca_correlation_graph/index.html
+++ b/api_modules/mlxtend.plotting/plot_pca_correlation_graph/index.html
@@ -12,25 +12,22 @@
         <link href="../../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../../cinder/css/base.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -668,14 +665,16 @@ <h2 id="plot_pca_correlation_graph">plot_pca_correlation_graph</h2>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../../mathjaxhelper.js" defer></script>
-        <script src="../../../search/main.js" defer></script>
+        <script src="../../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../../mathjaxhelper.js"></script>
+        <script src="../../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/api_modules/mlxtend.plotting/plot_sequential_feature_selection/index.html b/api_modules/mlxtend.plotting/plot_sequential_feature_selection/index.html
index fa78067e5..4f2d467eb 100644
--- a/api_modules/mlxtend.plotting/plot_sequential_feature_selection/index.html
+++ b/api_modules/mlxtend.plotting/plot_sequential_feature_selection/index.html
@@ -12,25 +12,22 @@
         <link href="../../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../../cinder/css/base.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -678,14 +675,16 @@ <h2 id="plot_sequential_feature_selection">plot_sequential_feature_selection</h2
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../../mathjaxhelper.js" defer></script>
-        <script src="../../../search/main.js" defer></script>
+        <script src="../../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../../mathjaxhelper.js"></script>
+        <script src="../../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/api_modules/mlxtend.plotting/remove_borders/index.html b/api_modules/mlxtend.plotting/remove_borders/index.html
index d506071ec..b851950df 100644
--- a/api_modules/mlxtend.plotting/remove_borders/index.html
+++ b/api_modules/mlxtend.plotting/remove_borders/index.html
@@ -12,25 +12,22 @@
         <link href="../../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../../cinder/css/base.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -657,14 +654,16 @@ <h2 id="remove_borders">remove_borders</h2>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../../mathjaxhelper.js" defer></script>
-        <script src="../../../search/main.js" defer></script>
+        <script src="../../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../../mathjaxhelper.js"></script>
+        <script src="../../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/api_modules/mlxtend.plotting/scatter_hist/index.html b/api_modules/mlxtend.plotting/scatter_hist/index.html
index 7353268b9..2f414863a 100644
--- a/api_modules/mlxtend.plotting/scatter_hist/index.html
+++ b/api_modules/mlxtend.plotting/scatter_hist/index.html
@@ -12,25 +12,22 @@
         <link href="../../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../../cinder/css/base.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -659,14 +656,16 @@ <h2 id="scatter_hist">scatter_hist</h2>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../../mathjaxhelper.js" defer></script>
-        <script src="../../../search/main.js" defer></script>
+        <script src="../../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../../mathjaxhelper.js"></script>
+        <script src="../../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/api_modules/mlxtend.plotting/scatterplotmatrix/index.html b/api_modules/mlxtend.plotting/scatterplotmatrix/index.html
index e30ce613a..caf0b7404 100644
--- a/api_modules/mlxtend.plotting/scatterplotmatrix/index.html
+++ b/api_modules/mlxtend.plotting/scatterplotmatrix/index.html
@@ -12,25 +12,22 @@
         <link href="../../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../../cinder/css/base.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -677,14 +674,16 @@ <h2 id="scatterplotmatrix">scatterplotmatrix</h2>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../../mathjaxhelper.js" defer></script>
-        <script src="../../../search/main.js" defer></script>
+        <script src="../../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../../mathjaxhelper.js"></script>
+        <script src="../../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/api_modules/mlxtend.plotting/stacked_barplot/index.html b/api_modules/mlxtend.plotting/stacked_barplot/index.html
index 95195a305..66f049f2e 100644
--- a/api_modules/mlxtend.plotting/stacked_barplot/index.html
+++ b/api_modules/mlxtend.plotting/stacked_barplot/index.html
@@ -12,25 +12,22 @@
         <link href="../../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../../cinder/css/base.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -663,14 +660,16 @@ <h2 id="stacked_barplot">stacked_barplot</h2>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../../mathjaxhelper.js" defer></script>
-        <script src="../../../search/main.js" defer></script>
+        <script src="../../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../../mathjaxhelper.js"></script>
+        <script src="../../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/api_modules/mlxtend.preprocessing/CopyTransformer/index.html b/api_modules/mlxtend.preprocessing/CopyTransformer/index.html
index cc1734843..653dca5b0 100644
--- a/api_modules/mlxtend.preprocessing/CopyTransformer/index.html
+++ b/api_modules/mlxtend.preprocessing/CopyTransformer/index.html
@@ -12,25 +12,22 @@
         <link href="../../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../../cinder/css/base.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -731,14 +728,16 @@ <h3 id="methods">Methods</h3>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../../mathjaxhelper.js" defer></script>
-        <script src="../../../search/main.js" defer></script>
+        <script src="../../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../../mathjaxhelper.js"></script>
+        <script src="../../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/api_modules/mlxtend.preprocessing/DenseTransformer/index.html b/api_modules/mlxtend.preprocessing/DenseTransformer/index.html
index 8df44d5b6..dbb5de089 100644
--- a/api_modules/mlxtend.preprocessing/DenseTransformer/index.html
+++ b/api_modules/mlxtend.preprocessing/DenseTransformer/index.html
@@ -12,25 +12,22 @@
         <link href="../../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../../cinder/css/base.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -731,14 +728,16 @@ <h3 id="methods">Methods</h3>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../../mathjaxhelper.js" defer></script>
-        <script src="../../../search/main.js" defer></script>
+        <script src="../../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../../mathjaxhelper.js"></script>
+        <script src="../../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/api_modules/mlxtend.preprocessing/MeanCenterer/index.html b/api_modules/mlxtend.preprocessing/MeanCenterer/index.html
index a30496c78..e25046d78 100644
--- a/api_modules/mlxtend.preprocessing/MeanCenterer/index.html
+++ b/api_modules/mlxtend.preprocessing/MeanCenterer/index.html
@@ -12,25 +12,22 @@
         <link href="../../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../../cinder/css/base.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -694,14 +691,16 @@ <h3 id="methods">Methods</h3>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../../mathjaxhelper.js" defer></script>
-        <script src="../../../search/main.js" defer></script>
+        <script src="../../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../../mathjaxhelper.js"></script>
+        <script src="../../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/api_modules/mlxtend.preprocessing/TransactionEncoder/index.html b/api_modules/mlxtend.preprocessing/TransactionEncoder/index.html
index 04c3fd94d..7ec4eeccd 100644
--- a/api_modules/mlxtend.preprocessing/TransactionEncoder/index.html
+++ b/api_modules/mlxtend.preprocessing/TransactionEncoder/index.html
@@ -12,25 +12,22 @@
         <link href="../../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../../cinder/css/base.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -823,14 +820,16 @@ <h3 id="methods">Methods</h3>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../../mathjaxhelper.js" defer></script>
-        <script src="../../../search/main.js" defer></script>
+        <script src="../../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../../mathjaxhelper.js"></script>
+        <script src="../../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/api_modules/mlxtend.preprocessing/minmax_scaling/index.html b/api_modules/mlxtend.preprocessing/minmax_scaling/index.html
index 903b0d008..0f4168511 100644
--- a/api_modules/mlxtend.preprocessing/minmax_scaling/index.html
+++ b/api_modules/mlxtend.preprocessing/minmax_scaling/index.html
@@ -12,25 +12,22 @@
         <link href="../../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../../cinder/css/base.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -659,14 +656,16 @@ <h2 id="minmax_scaling">minmax_scaling</h2>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../../mathjaxhelper.js" defer></script>
-        <script src="../../../search/main.js" defer></script>
+        <script src="../../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../../mathjaxhelper.js"></script>
+        <script src="../../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/api_modules/mlxtend.preprocessing/one_hot/index.html b/api_modules/mlxtend.preprocessing/one_hot/index.html
index dd94a6d90..d6e0d2472 100644
--- a/api_modules/mlxtend.preprocessing/one_hot/index.html
+++ b/api_modules/mlxtend.preprocessing/one_hot/index.html
@@ -12,25 +12,22 @@
         <link href="../../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../../cinder/css/base.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -657,14 +654,16 @@ <h2 id="one_hot">one_hot</h2>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../../mathjaxhelper.js" defer></script>
-        <script src="../../../search/main.js" defer></script>
+        <script src="../../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../../mathjaxhelper.js"></script>
+        <script src="../../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/api_modules/mlxtend.preprocessing/shuffle_arrays_unison/index.html b/api_modules/mlxtend.preprocessing/shuffle_arrays_unison/index.html
index 631bef703..176c4695d 100644
--- a/api_modules/mlxtend.preprocessing/shuffle_arrays_unison/index.html
+++ b/api_modules/mlxtend.preprocessing/shuffle_arrays_unison/index.html
@@ -12,25 +12,22 @@
         <link href="../../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../../cinder/css/base.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -660,14 +657,16 @@ <h2 id="shuffle_arrays_unison">shuffle_arrays_unison</h2>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../../mathjaxhelper.js" defer></script>
-        <script src="../../../search/main.js" defer></script>
+        <script src="../../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../../mathjaxhelper.js"></script>
+        <script src="../../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/api_modules/mlxtend.preprocessing/standardize/index.html b/api_modules/mlxtend.preprocessing/standardize/index.html
index 4590959eb..42a2d7ac5 100644
--- a/api_modules/mlxtend.preprocessing/standardize/index.html
+++ b/api_modules/mlxtend.preprocessing/standardize/index.html
@@ -12,25 +12,22 @@
         <link href="../../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../../cinder/css/base.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -676,14 +673,16 @@ <h2 id="standardize">standardize</h2>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../../mathjaxhelper.js" defer></script>
-        <script src="../../../search/main.js" defer></script>
+        <script src="../../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../../mathjaxhelper.js"></script>
+        <script src="../../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/api_modules/mlxtend.regressor/LinearRegression/index.html b/api_modules/mlxtend.regressor/LinearRegression/index.html
index 349966929..46adea4a3 100644
--- a/api_modules/mlxtend.regressor/LinearRegression/index.html
+++ b/api_modules/mlxtend.regressor/LinearRegression/index.html
@@ -12,25 +12,22 @@
         <link href="../../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../../cinder/css/base.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -778,14 +775,16 @@ <h3 id="methods">Methods</h3>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../../mathjaxhelper.js" defer></script>
-        <script src="../../../search/main.js" defer></script>
+        <script src="../../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../../mathjaxhelper.js"></script>
+        <script src="../../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/api_modules/mlxtend.regressor/StackingCVRegressor/index.html b/api_modules/mlxtend.regressor/StackingCVRegressor/index.html
index d0fd509d5..e309476d6 100644
--- a/api_modules/mlxtend.regressor/StackingCVRegressor/index.html
+++ b/api_modules/mlxtend.regressor/StackingCVRegressor/index.html
@@ -12,25 +12,22 @@
         <link href="../../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../../cinder/css/base.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -943,14 +940,16 @@ <h3 id="properties">Properties</h3>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../../mathjaxhelper.js" defer></script>
-        <script src="../../../search/main.js" defer></script>
+        <script src="../../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../../mathjaxhelper.js"></script>
+        <script src="../../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/api_modules/mlxtend.regressor/StackingRegressor/index.html b/api_modules/mlxtend.regressor/StackingRegressor/index.html
index 388e026c0..1b1607f11 100644
--- a/api_modules/mlxtend.regressor/StackingRegressor/index.html
+++ b/api_modules/mlxtend.regressor/StackingRegressor/index.html
@@ -12,25 +12,22 @@
         <link href="../../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../../cinder/css/base.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -905,14 +902,16 @@ <h3 id="properties">Properties</h3>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../../mathjaxhelper.js" defer></script>
-        <script src="../../../search/main.js" defer></script>
+        <script src="../../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../../mathjaxhelper.js"></script>
+        <script src="../../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/api_modules/mlxtend.text/generalize_names/index.html b/api_modules/mlxtend.text/generalize_names/index.html
index cb6498307..57aeb02f6 100644
--- a/api_modules/mlxtend.text/generalize_names/index.html
+++ b/api_modules/mlxtend.text/generalize_names/index.html
@@ -12,25 +12,22 @@
         <link href="../../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../../cinder/css/base.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -657,14 +654,16 @@ <h2 id="generalize_names">generalize_names</h2>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../../mathjaxhelper.js" defer></script>
-        <script src="../../../search/main.js" defer></script>
+        <script src="../../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../../mathjaxhelper.js"></script>
+        <script src="../../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/api_modules/mlxtend.text/generalize_names_duplcheck/index.html b/api_modules/mlxtend.text/generalize_names_duplcheck/index.html
index 1860511b3..d02c034c0 100644
--- a/api_modules/mlxtend.text/generalize_names_duplcheck/index.html
+++ b/api_modules/mlxtend.text/generalize_names_duplcheck/index.html
@@ -12,25 +12,22 @@
         <link href="../../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../../cinder/css/base.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -657,14 +654,16 @@ <h2 id="generalize_names_duplcheck">generalize_names_duplcheck</h2>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../../mathjaxhelper.js" defer></script>
-        <script src="../../../search/main.js" defer></script>
+        <script src="../../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../../mathjaxhelper.js"></script>
+        <script src="../../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/api_modules/mlxtend.text/tokenizer_emoticons/index.html b/api_modules/mlxtend.text/tokenizer_emoticons/index.html
index dc2c82617..7eae48d98 100644
--- a/api_modules/mlxtend.text/tokenizer_emoticons/index.html
+++ b/api_modules/mlxtend.text/tokenizer_emoticons/index.html
@@ -12,25 +12,22 @@
         <link href="../../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../../cinder/css/base.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -637,14 +634,16 @@ <h2 id="tokenizer_emoticons">tokenizer_emoticons</h2>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../../mathjaxhelper.js" defer></script>
-        <script src="../../../search/main.js" defer></script>
+        <script src="../../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../../mathjaxhelper.js"></script>
+        <script src="../../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/api_modules/mlxtend.text/tokenizer_words_and_emoticons/index.html b/api_modules/mlxtend.text/tokenizer_words_and_emoticons/index.html
index c6f293ae2..b8714c083 100644
--- a/api_modules/mlxtend.text/tokenizer_words_and_emoticons/index.html
+++ b/api_modules/mlxtend.text/tokenizer_words_and_emoticons/index.html
@@ -12,25 +12,22 @@
         <link href="../../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../../cinder/css/base.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -637,14 +634,16 @@ <h2 id="tokenizer_words_and_emoticons">tokenizer_words_and_emoticons</h2>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../../mathjaxhelper.js" defer></script>
-        <script src="../../../search/main.js" defer></script>
+        <script src="../../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../../mathjaxhelper.js"></script>
+        <script src="../../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/api_modules/mlxtend.utils/Counter/index.html b/api_modules/mlxtend.utils/Counter/index.html
index fc033d59f..e6f08ffde 100644
--- a/api_modules/mlxtend.utils/Counter/index.html
+++ b/api_modules/mlxtend.utils/Counter/index.html
@@ -12,25 +12,22 @@
         <link href="../../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../../cinder/css/base.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -687,14 +684,16 @@ <h3 id="methods">Methods</h3>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../../mathjaxhelper.js" defer></script>
-        <script src="../../../search/main.js" defer></script>
+        <script src="../../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../../mathjaxhelper.js"></script>
+        <script src="../../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/api_modules/mlxtend.utils/assert_raises/index.html b/api_modules/mlxtend.utils/assert_raises/index.html
index 70e138cf9..4702e9b5e 100644
--- a/api_modules/mlxtend.utils/assert_raises/index.html
+++ b/api_modules/mlxtend.utils/assert_raises/index.html
@@ -12,25 +12,22 @@
         <link href="../../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../../cinder/css/base.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -651,14 +648,16 @@ <h2 id="assert_raises">assert_raises</h2>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../../mathjaxhelper.js" defer></script>
-        <script src="../../../search/main.js" defer></script>
+        <script src="../../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../../mathjaxhelper.js"></script>
+        <script src="../../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/api_modules/mlxtend.utils/check_Xy/index.html b/api_modules/mlxtend.utils/check_Xy/index.html
index 55491daae..c46ba6810 100644
--- a/api_modules/mlxtend.utils/check_Xy/index.html
+++ b/api_modules/mlxtend.utils/check_Xy/index.html
@@ -12,25 +12,22 @@
         <link href="../../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../../cinder/css/base.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -630,14 +627,16 @@ <h2 id="check_xy">check_Xy</h2>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../../mathjaxhelper.js" defer></script>
-        <script src="../../../search/main.js" defer></script>
+        <script src="../../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../../mathjaxhelper.js"></script>
+        <script src="../../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/api_modules/mlxtend.utils/format_kwarg_dictionaries/index.html b/api_modules/mlxtend.utils/format_kwarg_dictionaries/index.html
index 8a72e0929..c2bd99ae0 100644
--- a/api_modules/mlxtend.utils/format_kwarg_dictionaries/index.html
+++ b/api_modules/mlxtend.utils/format_kwarg_dictionaries/index.html
@@ -12,25 +12,22 @@
         <link href="../../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../../cinder/css/base.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -653,14 +650,16 @@ <h2 id="format_kwarg_dictionaries">format_kwarg_dictionaries</h2>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../../mathjaxhelper.js" defer></script>
-        <script src="../../../search/main.js" defer></script>
+        <script src="../../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../../mathjaxhelper.js"></script>
+        <script src="../../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/api_subpackages/mlxtend._base/index.html b/api_subpackages/mlxtend._base/index.html
index 506d8dff6..12a6f2102 100644
--- a/api_subpackages/mlxtend._base/index.html
+++ b/api_subpackages/mlxtend._base/index.html
@@ -12,25 +12,22 @@
         <link href="../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../cinder/css/base.css" rel="stylesheet">
         <link href="../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -623,14 +620,16 @@
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../mathjaxhelper.js" defer></script>
-        <script src="../../search/main.js" defer></script>
+        <script src="../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../mathjaxhelper.js"></script>
+        <script src="../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/api_subpackages/mlxtend.classifier/index.html b/api_subpackages/mlxtend.classifier/index.html
index 1aae31985..9fbd9393a 100644
--- a/api_subpackages/mlxtend.classifier/index.html
+++ b/api_subpackages/mlxtend.classifier/index.html
@@ -12,25 +12,22 @@
         <link href="../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../cinder/css/base.css" rel="stylesheet">
         <link href="../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -2722,14 +2719,16 @@ <h3 id="properties_1">Properties</h3>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../mathjaxhelper.js" defer></script>
-        <script src="../../search/main.js" defer></script>
+        <script src="../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../mathjaxhelper.js"></script>
+        <script src="../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/api_subpackages/mlxtend.cluster/index.html b/api_subpackages/mlxtend.cluster/index.html
index a23322467..3a88aa2ad 100644
--- a/api_subpackages/mlxtend.cluster/index.html
+++ b/api_subpackages/mlxtend.cluster/index.html
@@ -12,25 +12,22 @@
         <link href="../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../cinder/css/base.css" rel="stylesheet">
         <link href="../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -775,14 +772,16 @@ <h3 id="methods">Methods</h3>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../mathjaxhelper.js" defer></script>
-        <script src="../../search/main.js" defer></script>
+        <script src="../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../mathjaxhelper.js"></script>
+        <script src="../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/api_subpackages/mlxtend.data/index.html b/api_subpackages/mlxtend.data/index.html
index 5c69c0699..0aa714a41 100644
--- a/api_subpackages/mlxtend.data/index.html
+++ b/api_subpackages/mlxtend.data/index.html
@@ -12,25 +12,22 @@
         <link href="../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../cinder/css/base.css" rel="stylesheet">
         <link href="../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -983,14 +980,16 @@ <h2 id="wine_data">wine_data</h2>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../mathjaxhelper.js" defer></script>
-        <script src="../../search/main.js" defer></script>
+        <script src="../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../mathjaxhelper.js"></script>
+        <script src="../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/api_subpackages/mlxtend.evaluate/index.html b/api_subpackages/mlxtend.evaluate/index.html
index 51fff6156..caa72cddd 100644
--- a/api_subpackages/mlxtend.evaluate/index.html
+++ b/api_subpackages/mlxtend.evaluate/index.html
@@ -12,25 +12,22 @@
         <link href="../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../cinder/css/base.css" rel="stylesheet">
         <link href="../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -2132,14 +2129,16 @@ <h2 id="scoring">scoring</h2>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../mathjaxhelper.js" defer></script>
-        <script src="../../search/main.js" defer></script>
+        <script src="../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../mathjaxhelper.js"></script>
+        <script src="../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/api_subpackages/mlxtend.externals/index.html b/api_subpackages/mlxtend.externals/index.html
index f98e17e35..dbf149711 100644
--- a/api_subpackages/mlxtend.externals/index.html
+++ b/api_subpackages/mlxtend.externals/index.html
@@ -12,25 +12,22 @@
         <link href="../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../cinder/css/base.css" rel="stylesheet">
         <link href="../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -623,14 +620,16 @@
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../mathjaxhelper.js" defer></script>
-        <script src="../../search/main.js" defer></script>
+        <script src="../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../mathjaxhelper.js"></script>
+        <script src="../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/api_subpackages/mlxtend.feature_extraction/index.html b/api_subpackages/mlxtend.feature_extraction/index.html
index a8d787b3d..2f9e8f24e 100644
--- a/api_subpackages/mlxtend.feature_extraction/index.html
+++ b/api_subpackages/mlxtend.feature_extraction/index.html
@@ -12,25 +12,22 @@
         <link href="../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../cinder/css/base.css" rel="stylesheet">
         <link href="../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -1003,14 +1000,16 @@ <h3 id="methods_2">Methods</h3>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../mathjaxhelper.js" defer></script>
-        <script src="../../search/main.js" defer></script>
+        <script src="../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../mathjaxhelper.js"></script>
+        <script src="../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/api_subpackages/mlxtend.feature_selection/index.html b/api_subpackages/mlxtend.feature_selection/index.html
index 0aed745e1..9b7ca2e24 100644
--- a/api_subpackages/mlxtend.feature_selection/index.html
+++ b/api_subpackages/mlxtend.feature_selection/index.html
@@ -12,25 +12,22 @@
         <link href="../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../cinder/css/base.css" rel="stylesheet">
         <link href="../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -1391,14 +1388,16 @@ <h3 id="properties">Properties</h3>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../mathjaxhelper.js" defer></script>
-        <script src="../../search/main.js" defer></script>
+        <script src="../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../mathjaxhelper.js"></script>
+        <script src="../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/api_subpackages/mlxtend.file_io/index.html b/api_subpackages/mlxtend.file_io/index.html
index e0721ecd1..dfab77868 100644
--- a/api_subpackages/mlxtend.file_io/index.html
+++ b/api_subpackages/mlxtend.file_io/index.html
@@ -12,25 +12,22 @@
         <link href="../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../cinder/css/base.css" rel="stylesheet">
         <link href="../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -740,14 +737,16 @@ <h2 id="find_files">find_files</h2>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../mathjaxhelper.js" defer></script>
-        <script src="../../search/main.js" defer></script>
+        <script src="../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../mathjaxhelper.js"></script>
+        <script src="../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/api_subpackages/mlxtend.frequent_patterns/index.html b/api_subpackages/mlxtend.frequent_patterns/index.html
index 7a2b44e8a..bbf81dfb1 100644
--- a/api_subpackages/mlxtend.frequent_patterns/index.html
+++ b/api_subpackages/mlxtend.frequent_patterns/index.html
@@ -12,25 +12,22 @@
         <link href="../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../cinder/css/base.css" rel="stylesheet">
         <link href="../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -970,14 +967,16 @@ <h2 id="hmine">hmine</h2>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../mathjaxhelper.js" defer></script>
-        <script src="../../search/main.js" defer></script>
+        <script src="../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../mathjaxhelper.js"></script>
+        <script src="../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/api_subpackages/mlxtend.image/index.html b/api_subpackages/mlxtend.image/index.html
index a9e57db0e..cf140a221 100644
--- a/api_subpackages/mlxtend.image/index.html
+++ b/api_subpackages/mlxtend.image/index.html
@@ -12,25 +12,22 @@
         <link href="../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../cinder/css/base.css" rel="stylesheet">
         <link href="../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -815,14 +812,16 @@ <h2 id="extract_face_landmarks">extract_face_landmarks</h2>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../mathjaxhelper.js" defer></script>
-        <script src="../../search/main.js" defer></script>
+        <script src="../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../mathjaxhelper.js"></script>
+        <script src="../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/api_subpackages/mlxtend.math/index.html b/api_subpackages/mlxtend.math/index.html
index 09b3410d9..9f7eaf574 100644
--- a/api_subpackages/mlxtend.math/index.html
+++ b/api_subpackages/mlxtend.math/index.html
@@ -12,25 +12,22 @@
         <link href="../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../cinder/css/base.css" rel="stylesheet">
         <link href="../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -757,14 +754,16 @@ <h2 id="vectorspace_orthonormalization">vectorspace_orthonormalization</h2>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../mathjaxhelper.js" defer></script>
-        <script src="../../search/main.js" defer></script>
+        <script src="../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../mathjaxhelper.js"></script>
+        <script src="../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/api_subpackages/mlxtend.plotting/index.html b/api_subpackages/mlxtend.plotting/index.html
index 598da4f15..592843ae2 100644
--- a/api_subpackages/mlxtend.plotting/index.html
+++ b/api_subpackages/mlxtend.plotting/index.html
@@ -12,25 +12,22 @@
         <link href="../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../cinder/css/base.css" rel="stylesheet">
         <link href="../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -1552,14 +1549,16 @@ <h2 id="stacked_barplot">stacked_barplot</h2>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../mathjaxhelper.js" defer></script>
-        <script src="../../search/main.js" defer></script>
+        <script src="../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../mathjaxhelper.js"></script>
+        <script src="../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/api_subpackages/mlxtend.preprocessing/index.html b/api_subpackages/mlxtend.preprocessing/index.html
index 4d1d2bddb..f041fd8fc 100644
--- a/api_subpackages/mlxtend.preprocessing/index.html
+++ b/api_subpackages/mlxtend.preprocessing/index.html
@@ -12,25 +12,22 @@
         <link href="../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../cinder/css/base.css" rel="stylesheet">
         <link href="../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -1288,14 +1285,16 @@ <h2 id="standardize">standardize</h2>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../mathjaxhelper.js" defer></script>
-        <script src="../../search/main.js" defer></script>
+        <script src="../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../mathjaxhelper.js"></script>
+        <script src="../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/api_subpackages/mlxtend.regressor/index.html b/api_subpackages/mlxtend.regressor/index.html
index 6b8168027..c27db3716 100644
--- a/api_subpackages/mlxtend.regressor/index.html
+++ b/api_subpackages/mlxtend.regressor/index.html
@@ -12,25 +12,22 @@
         <link href="../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../cinder/css/base.css" rel="stylesheet">
         <link href="../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -1393,14 +1390,16 @@ <h3 id="properties_1">Properties</h3>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../mathjaxhelper.js" defer></script>
-        <script src="../../search/main.js" defer></script>
+        <script src="../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../mathjaxhelper.js"></script>
+        <script src="../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/api_subpackages/mlxtend.text/index.html b/api_subpackages/mlxtend.text/index.html
index 7a709461a..b53174989 100644
--- a/api_subpackages/mlxtend.text/index.html
+++ b/api_subpackages/mlxtend.text/index.html
@@ -12,25 +12,22 @@
         <link href="../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../cinder/css/base.css" rel="stylesheet">
         <link href="../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -733,14 +730,16 @@ <h2 id="tokenizer_words_and_emoticons">tokenizer_words_and_emoticons</h2>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../mathjaxhelper.js" defer></script>
-        <script src="../../search/main.js" defer></script>
+        <script src="../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../mathjaxhelper.js"></script>
+        <script src="../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/api_subpackages/mlxtend.utils/index.html b/api_subpackages/mlxtend.utils/index.html
index 61c091603..faf6e4af1 100644
--- a/api_subpackages/mlxtend.utils/index.html
+++ b/api_subpackages/mlxtend.utils/index.html
@@ -12,25 +12,22 @@
         <link href="../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../cinder/css/base.css" rel="stylesheet">
         <link href="../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -766,14 +763,16 @@ <h2 id="format_kwarg_dictionaries">format_kwarg_dictionaries</h2>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../mathjaxhelper.js" defer></script>
-        <script src="../../search/main.js" defer></script>
+        <script src="../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../mathjaxhelper.js"></script>
+        <script src="../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/cite/index.html b/cite/index.html
index ae952af85..b0c2d7e56 100644
--- a/cite/index.html
+++ b/cite/index.html
@@ -12,25 +12,22 @@
         <link href="../css/bootstrap.min.css" rel="stylesheet">
         <link href="../css/font-awesome.min.css" rel="stylesheet">
         <link href="../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../cinder/css/base.css" rel="stylesheet">
         <link href="../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../cinder/css/cinder.css" rel="stylesheet">
         <link href="../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -659,14 +656,16 @@ <h1 id="citing-mlxtend">Citing mlxtend</h1>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../js/jquery-3.6.0.min.js"></script>
+        <script src="../js/bootstrap.min.js"></script>
         <script>
             var base_url = "..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../mathjaxhelper.js" defer></script>
-        <script src="../search/main.js" defer></script>
+        <script src="../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../mathjaxhelper.js"></script>
+        <script src="../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/contributors/index.html b/contributors/index.html
index f806315dd..2d223a609 100644
--- a/contributors/index.html
+++ b/contributors/index.html
@@ -12,25 +12,22 @@
         <link href="../css/bootstrap.min.css" rel="stylesheet">
         <link href="../css/font-awesome.min.css" rel="stylesheet">
         <link href="../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../cinder/css/base.css" rel="stylesheet">
         <link href="../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../cinder/css/cinder.css" rel="stylesheet">
         <link href="../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -639,14 +636,16 @@ <h1 id="contributors">Contributors</h1>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../js/jquery-3.6.0.min.js"></script>
+        <script src="../js/bootstrap.min.js"></script>
         <script>
             var base_url = "..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../mathjaxhelper.js" defer></script>
-        <script src="../search/main.js" defer></script>
+        <script src="../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../mathjaxhelper.js"></script>
+        <script src="../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/discuss/index.html b/discuss/index.html
index b3b1e013d..49ea52fbe 100644
--- a/discuss/index.html
+++ b/discuss/index.html
@@ -12,25 +12,22 @@
         <link href="../css/bootstrap.min.css" rel="stylesheet">
         <link href="../css/font-awesome.min.css" rel="stylesheet">
         <link href="../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../cinder/css/base.css" rel="stylesheet">
         <link href="../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../cinder/css/cinder.css" rel="stylesheet">
         <link href="../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -640,14 +637,16 @@ <h1 id="discuss">Discuss</h1>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../js/jquery-3.6.0.min.js"></script>
+        <script src="../js/bootstrap.min.js"></script>
         <script>
             var base_url = "..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../mathjaxhelper.js" defer></script>
-        <script src="../search/main.js" defer></script>
+        <script src="../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../mathjaxhelper.js"></script>
+        <script src="../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/index.html b/index.html
index e3ad9669c..b84854138 100644
--- a/index.html
+++ b/index.html
@@ -12,25 +12,22 @@
         <link href="css/bootstrap.min.css" rel="stylesheet">
         <link href="css/font-awesome.min.css" rel="stylesheet">
         <link href="css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="cinder/css/base.css" rel="stylesheet">
         <link href="cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="cinder/css/cinder.css" rel="stylesheet">
         <link href="cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="js/jquery-1.10.2.min.js" defer></script>
-        <script src="js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -746,14 +743,16 @@ <h2 id="contact">Contact</h2>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="js/jquery-3.6.0.min.js"></script>
+        <script src="js/bootstrap.min.js"></script>
         <script>
             var base_url = ".",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="mathjaxhelper.js" defer></script>
-        <script src="search/main.js" defer></script>
+        <script src="js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="mathjaxhelper.js"></script>
+        <script src="search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
@@ -820,6 +819,6 @@ <h4 class="modal-title" id="keyboardModalLabel">Keyboard Shortcuts</h4>
 </html>
 
 <!--
-MkDocs version : 1.4.2
-Build Date UTC : 2024-11-05 14:13:52.486276+00:00
+MkDocs version : 1.5.3
+Build Date UTC : 2024-11-15 00:33:35.332772+00:00
 -->
diff --git a/installation/index.html b/installation/index.html
index e23e37203..244443273 100644
--- a/installation/index.html
+++ b/installation/index.html
@@ -12,25 +12,22 @@
         <link href="../css/bootstrap.min.css" rel="stylesheet">
         <link href="../css/font-awesome.min.css" rel="stylesheet">
         <link href="../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../cinder/css/base.css" rel="stylesheet">
         <link href="../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../cinder/css/cinder.css" rel="stylesheet">
         <link href="../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -672,14 +669,16 @@ <h3 id="dev-version">Dev Version</h3>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../js/jquery-3.6.0.min.js"></script>
+        <script src="../js/bootstrap.min.js"></script>
         <script>
             var base_url = "..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../mathjaxhelper.js" defer></script>
-        <script src="../search/main.js" defer></script>
+        <script src="../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../mathjaxhelper.js"></script>
+        <script src="../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/js/jquery-1.10.2.min.js b/js/jquery-1.10.2.min.js
deleted file mode 100644
index da4170647..000000000
--- a/js/jquery-1.10.2.min.js
+++ /dev/null
@@ -1,6 +0,0 @@
-/*! jQuery v1.10.2 | (c) 2005, 2013 jQuery Foundation, Inc. | jquery.org/license
-//@ sourceMappingURL=jquery-1.10.2.min.map
-*/
-(function(e,t){var n,r,i=typeof t,o=e.location,a=e.document,s=a.documentElement,l=e.jQuery,u=e.$,c={},p=[],f="1.10.2",d=p.concat,h=p.push,g=p.slice,m=p.indexOf,y=c.toString,v=c.hasOwnProperty,b=f.trim,x=function(e,t){return new x.fn.init(e,t,r)},w=/[+-]?(?:\d*\.|)\d+(?:[eE][+-]?\d+|)/.source,T=/\S+/g,C=/^[\s\uFEFF\xA0]+|[\s\uFEFF\xA0]+$/g,N=/^(?:\s*(<[\w\W]+>)[^>]*|#([\w-]*))$/,k=/^<(\w+)\s*\/?>(?:<\/\1>|)$/,E=/^[\],:{}\s]*$/,S=/(?:^|:|,)(?:\s*\[)+/g,A=/\\(?:["\\\/bfnrt]|u[\da-fA-F]{4})/g,j=/"[^"\\\r\n]*"|true|false|null|-?(?:\d+\.|)\d+(?:[eE][+-]?\d+|)/g,D=/^-ms-/,L=/-([\da-z])/gi,H=function(e,t){return t.toUpperCase()},q=function(e){(a.addEventListener||"load"===e.type||"complete"===a.readyState)&&(_(),x.ready())},_=function(){a.addEventListener?(a.removeEventListener("DOMContentLoaded",q,!1),e.removeEventListener("load",q,!1)):(a.detachEvent("onreadystatechange",q),e.detachEvent("onload",q))};x.fn=x.prototype={jquery:f,constructor:x,init:function(e,n,r){var i,o;if(!e)return this;if("string"==typeof e){if(i="<"===e.charAt(0)&&">"===e.charAt(e.length-1)&&e.length>=3?[null,e,null]:N.exec(e),!i||!i[1]&&n)return!n||n.jquery?(n||r).find(e):this.constructor(n).find(e);if(i[1]){if(n=n instanceof x?n[0]:n,x.merge(this,x.parseHTML(i[1],n&&n.nodeType?n.ownerDocument||n:a,!0)),k.test(i[1])&&x.isPlainObject(n))for(i in n)x.isFunction(this[i])?this[i](n[i]):this.attr(i,n[i]);return this}if(o=a.getElementById(i[2]),o&&o.parentNode){if(o.id!==i[2])return r.find(e);this.length=1,this[0]=o}return this.context=a,this.selector=e,this}return e.nodeType?(this.context=this[0]=e,this.length=1,this):x.isFunction(e)?r.ready(e):(e.selector!==t&&(this.selector=e.selector,this.context=e.context),x.makeArray(e,this))},selector:"",length:0,toArray:function(){return g.call(this)},get:function(e){return null==e?this.toArray():0>e?this[this.length+e]:this[e]},pushStack:function(e){var t=x.merge(this.constructor(),e);return t.prevObject=this,t.context=this.context,t},each:function(e,t){return x.each(this,e,t)},ready:function(e){return x.ready.promise().done(e),this},slice:function(){return this.pushStack(g.apply(this,arguments))},first:function(){return this.eq(0)},last:function(){return this.eq(-1)},eq:function(e){var t=this.length,n=+e+(0>e?t:0);return this.pushStack(n>=0&&t>n?[this[n]]:[])},map:function(e){return this.pushStack(x.map(this,function(t,n){return e.call(t,n,t)}))},end:function(){return this.prevObject||this.constructor(null)},push:h,sort:[].sort,splice:[].splice},x.fn.init.prototype=x.fn,x.extend=x.fn.extend=function(){var e,n,r,i,o,a,s=arguments[0]||{},l=1,u=arguments.length,c=!1;for("boolean"==typeof s&&(c=s,s=arguments[1]||{},l=2),"object"==typeof s||x.isFunction(s)||(s={}),u===l&&(s=this,--l);u>l;l++)if(null!=(o=arguments[l]))for(i in o)e=s[i],r=o[i],s!==r&&(c&&r&&(x.isPlainObject(r)||(n=x.isArray(r)))?(n?(n=!1,a=e&&x.isArray(e)?e:[]):a=e&&x.isPlainObject(e)?e:{},s[i]=x.extend(c,a,r)):r!==t&&(s[i]=r));return s},x.extend({expando:"jQuery"+(f+Math.random()).replace(/\D/g,""),noConflict:function(t){return e.$===x&&(e.$=u),t&&e.jQuery===x&&(e.jQuery=l),x},isReady:!1,readyWait:1,holdReady:function(e){e?x.readyWait++:x.ready(!0)},ready:function(e){if(e===!0?!--x.readyWait:!x.isReady){if(!a.body)return setTimeout(x.ready);x.isReady=!0,e!==!0&&--x.readyWait>0||(n.resolveWith(a,[x]),x.fn.trigger&&x(a).trigger("ready").off("ready"))}},isFunction:function(e){return"function"===x.type(e)},isArray:Array.isArray||function(e){return"array"===x.type(e)},isWindow:function(e){return null!=e&&e==e.window},isNumeric:function(e){return!isNaN(parseFloat(e))&&isFinite(e)},type:function(e){return null==e?e+"":"object"==typeof e||"function"==typeof e?c[y.call(e)]||"object":typeof e},isPlainObject:function(e){var n;if(!e||"object"!==x.type(e)||e.nodeType||x.isWindow(e))return!1;try{if(e.constructor&&!v.call(e,"constructor")&&!v.call(e.constructor.prototype,"isPrototypeOf"))return!1}catch(r){return!1}if(x.support.ownLast)for(n in e)return v.call(e,n);for(n in e);return n===t||v.call(e,n)},isEmptyObject:function(e){var t;for(t in e)return!1;return!0},error:function(e){throw Error(e)},parseHTML:function(e,t,n){if(!e||"string"!=typeof e)return null;"boolean"==typeof t&&(n=t,t=!1),t=t||a;var r=k.exec(e),i=!n&&[];return r?[t.createElement(r[1])]:(r=x.buildFragment([e],t,i),i&&x(i).remove(),x.merge([],r.childNodes))},parseJSON:function(n){return e.JSON&&e.JSON.parse?e.JSON.parse(n):null===n?n:"string"==typeof n&&(n=x.trim(n),n&&E.test(n.replace(A,"@").replace(j,"]").replace(S,"")))?Function("return "+n)():(x.error("Invalid JSON: "+n),t)},parseXML:function(n){var r,i;if(!n||"string"!=typeof n)return null;try{e.DOMParser?(i=new DOMParser,r=i.parseFromString(n,"text/xml")):(r=new ActiveXObject("Microsoft.XMLDOM"),r.async="false",r.loadXML(n))}catch(o){r=t}return r&&r.documentElement&&!r.getElementsByTagName("parsererror").length||x.error("Invalid XML: "+n),r},noop:function(){},globalEval:function(t){t&&x.trim(t)&&(e.execScript||function(t){e.eval.call(e,t)})(t)},camelCase:function(e){return e.replace(D,"ms-").replace(L,H)},nodeName:function(e,t){return e.nodeName&&e.nodeName.toLowerCase()===t.toLowerCase()},each:function(e,t,n){var r,i=0,o=e.length,a=M(e);if(n){if(a){for(;o>i;i++)if(r=t.apply(e[i],n),r===!1)break}else for(i in e)if(r=t.apply(e[i],n),r===!1)break}else if(a){for(;o>i;i++)if(r=t.call(e[i],i,e[i]),r===!1)break}else for(i in e)if(r=t.call(e[i],i,e[i]),r===!1)break;return e},trim:b&&!b.call("\ufeff\u00a0")?function(e){return null==e?"":b.call(e)}:function(e){return null==e?"":(e+"").replace(C,"")},makeArray:function(e,t){var n=t||[];return null!=e&&(M(Object(e))?x.merge(n,"string"==typeof e?[e]:e):h.call(n,e)),n},inArray:function(e,t,n){var r;if(t){if(m)return m.call(t,e,n);for(r=t.length,n=n?0>n?Math.max(0,r+n):n:0;r>n;n++)if(n in t&&t[n]===e)return n}return-1},merge:function(e,n){var r=n.length,i=e.length,o=0;if("number"==typeof r)for(;r>o;o++)e[i++]=n[o];else while(n[o]!==t)e[i++]=n[o++];return e.length=i,e},grep:function(e,t,n){var r,i=[],o=0,a=e.length;for(n=!!n;a>o;o++)r=!!t(e[o],o),n!==r&&i.push(e[o]);return i},map:function(e,t,n){var r,i=0,o=e.length,a=M(e),s=[];if(a)for(;o>i;i++)r=t(e[i],i,n),null!=r&&(s[s.length]=r);else for(i in e)r=t(e[i],i,n),null!=r&&(s[s.length]=r);return d.apply([],s)},guid:1,proxy:function(e,n){var r,i,o;return"string"==typeof n&&(o=e[n],n=e,e=o),x.isFunction(e)?(r=g.call(arguments,2),i=function(){return e.apply(n||this,r.concat(g.call(arguments)))},i.guid=e.guid=e.guid||x.guid++,i):t},access:function(e,n,r,i,o,a,s){var l=0,u=e.length,c=null==r;if("object"===x.type(r)){o=!0;for(l in r)x.access(e,n,l,r[l],!0,a,s)}else if(i!==t&&(o=!0,x.isFunction(i)||(s=!0),c&&(s?(n.call(e,i),n=null):(c=n,n=function(e,t,n){return c.call(x(e),n)})),n))for(;u>l;l++)n(e[l],r,s?i:i.call(e[l],l,n(e[l],r)));return o?e:c?n.call(e):u?n(e[0],r):a},now:function(){return(new Date).getTime()},swap:function(e,t,n,r){var i,o,a={};for(o in t)a[o]=e.style[o],e.style[o]=t[o];i=n.apply(e,r||[]);for(o in t)e.style[o]=a[o];return i}}),x.ready.promise=function(t){if(!n)if(n=x.Deferred(),"complete"===a.readyState)setTimeout(x.ready);else if(a.addEventListener)a.addEventListener("DOMContentLoaded",q,!1),e.addEventListener("load",q,!1);else{a.attachEvent("onreadystatechange",q),e.attachEvent("onload",q);var r=!1;try{r=null==e.frameElement&&a.documentElement}catch(i){}r&&r.doScroll&&function o(){if(!x.isReady){try{r.doScroll("left")}catch(e){return setTimeout(o,50)}_(),x.ready()}}()}return n.promise(t)},x.each("Boolean Number String Function Array Date RegExp Object Error".split(" "),function(e,t){c["[object "+t+"]"]=t.toLowerCase()});function M(e){var t=e.length,n=x.type(e);return x.isWindow(e)?!1:1===e.nodeType&&t?!0:"array"===n||"function"!==n&&(0===t||"number"==typeof t&&t>0&&t-1 in e)}r=x(a),function(e,t){var n,r,i,o,a,s,l,u,c,p,f,d,h,g,m,y,v,b="sizzle"+-new Date,w=e.document,T=0,C=0,N=st(),k=st(),E=st(),S=!1,A=function(e,t){return e===t?(S=!0,0):0},j=typeof t,D=1<<31,L={}.hasOwnProperty,H=[],q=H.pop,_=H.push,M=H.push,O=H.slice,F=H.indexOf||function(e){var t=0,n=this.length;for(;n>t;t++)if(this[t]===e)return t;return-1},B="checked|selected|async|autofocus|autoplay|controls|defer|disabled|hidden|ismap|loop|multiple|open|readonly|required|scoped",P="[\\x20\\t\\r\\n\\f]",R="(?:\\\\.|[\\w-]|[^\\x00-\\xa0])+",W=R.replace("w","w#"),$="\\["+P+"*("+R+")"+P+"*(?:([*^$|!~]?=)"+P+"*(?:(['\"])((?:\\\\.|[^\\\\])*?)\\3|("+W+")|)|)"+P+"*\\]",I=":("+R+")(?:\\(((['\"])((?:\\\\.|[^\\\\])*?)\\3|((?:\\\\.|[^\\\\()[\\]]|"+$.replace(3,8)+")*)|.*)\\)|)",z=RegExp("^"+P+"+|((?:^|[^\\\\])(?:\\\\.)*)"+P+"+$","g"),X=RegExp("^"+P+"*,"+P+"*"),U=RegExp("^"+P+"*([>+~]|"+P+")"+P+"*"),V=RegExp(P+"*[+~]"),Y=RegExp("="+P+"*([^\\]'\"]*)"+P+"*\\]","g"),J=RegExp(I),G=RegExp("^"+W+"$"),Q={ID:RegExp("^#("+R+")"),CLASS:RegExp("^\\.("+R+")"),TAG:RegExp("^("+R.replace("w","w*")+")"),ATTR:RegExp("^"+$),PSEUDO:RegExp("^"+I),CHILD:RegExp("^:(only|first|last|nth|nth-last)-(child|of-type)(?:\\("+P+"*(even|odd|(([+-]|)(\\d*)n|)"+P+"*(?:([+-]|)"+P+"*(\\d+)|))"+P+"*\\)|)","i"),bool:RegExp("^(?:"+B+")$","i"),needsContext:RegExp("^"+P+"*[>+~]|:(even|odd|eq|gt|lt|nth|first|last)(?:\\("+P+"*((?:-\\d)?\\d*)"+P+"*\\)|)(?=[^-]|$)","i")},K=/^[^{]+\{\s*\[native \w/,Z=/^(?:#([\w-]+)|(\w+)|\.([\w-]+))$/,et=/^(?:input|select|textarea|button)$/i,tt=/^h\d$/i,nt=/'|\\/g,rt=RegExp("\\\\([\\da-f]{1,6}"+P+"?|("+P+")|.)","ig"),it=function(e,t,n){var r="0x"+t-65536;return r!==r||n?t:0>r?String.fromCharCode(r+65536):String.fromCharCode(55296|r>>10,56320|1023&r)};try{M.apply(H=O.call(w.childNodes),w.childNodes),H[w.childNodes.length].nodeType}catch(ot){M={apply:H.length?function(e,t){_.apply(e,O.call(t))}:function(e,t){var n=e.length,r=0;while(e[n++]=t[r++]);e.length=n-1}}}function at(e,t,n,i){var o,a,s,l,u,c,d,m,y,x;if((t?t.ownerDocument||t:w)!==f&&p(t),t=t||f,n=n||[],!e||"string"!=typeof e)return n;if(1!==(l=t.nodeType)&&9!==l)return[];if(h&&!i){if(o=Z.exec(e))if(s=o[1]){if(9===l){if(a=t.getElementById(s),!a||!a.parentNode)return n;if(a.id===s)return n.push(a),n}else if(t.ownerDocument&&(a=t.ownerDocument.getElementById(s))&&v(t,a)&&a.id===s)return n.push(a),n}else{if(o[2])return M.apply(n,t.getElementsByTagName(e)),n;if((s=o[3])&&r.getElementsByClassName&&t.getElementsByClassName)return M.apply(n,t.getElementsByClassName(s)),n}if(r.qsa&&(!g||!g.test(e))){if(m=d=b,y=t,x=9===l&&e,1===l&&"object"!==t.nodeName.toLowerCase()){c=mt(e),(d=t.getAttribute("id"))?m=d.replace(nt,"\\$&"):t.setAttribute("id",m),m="[id='"+m+"'] ",u=c.length;while(u--)c[u]=m+yt(c[u]);y=V.test(e)&&t.parentNode||t,x=c.join(",")}if(x)try{return M.apply(n,y.querySelectorAll(x)),n}catch(T){}finally{d||t.removeAttribute("id")}}}return kt(e.replace(z,"$1"),t,n,i)}function st(){var e=[];function t(n,r){return e.push(n+=" ")>o.cacheLength&&delete t[e.shift()],t[n]=r}return t}function lt(e){return e[b]=!0,e}function ut(e){var t=f.createElement("div");try{return!!e(t)}catch(n){return!1}finally{t.parentNode&&t.parentNode.removeChild(t),t=null}}function ct(e,t){var n=e.split("|"),r=e.length;while(r--)o.attrHandle[n[r]]=t}function pt(e,t){var n=t&&e,r=n&&1===e.nodeType&&1===t.nodeType&&(~t.sourceIndex||D)-(~e.sourceIndex||D);if(r)return r;if(n)while(n=n.nextSibling)if(n===t)return-1;return e?1:-1}function ft(e){return function(t){var n=t.nodeName.toLowerCase();return"input"===n&&t.type===e}}function dt(e){return function(t){var n=t.nodeName.toLowerCase();return("input"===n||"button"===n)&&t.type===e}}function ht(e){return lt(function(t){return t=+t,lt(function(n,r){var i,o=e([],n.length,t),a=o.length;while(a--)n[i=o[a]]&&(n[i]=!(r[i]=n[i]))})})}s=at.isXML=function(e){var t=e&&(e.ownerDocument||e).documentElement;return t?"HTML"!==t.nodeName:!1},r=at.support={},p=at.setDocument=function(e){var n=e?e.ownerDocument||e:w,i=n.defaultView;return n!==f&&9===n.nodeType&&n.documentElement?(f=n,d=n.documentElement,h=!s(n),i&&i.attachEvent&&i!==i.top&&i.attachEvent("onbeforeunload",function(){p()}),r.attributes=ut(function(e){return e.className="i",!e.getAttribute("className")}),r.getElementsByTagName=ut(function(e){return e.appendChild(n.createComment("")),!e.getElementsByTagName("*").length}),r.getElementsByClassName=ut(function(e){return e.innerHTML="<div class='a'></div><div class='a i'></div>",e.firstChild.className="i",2===e.getElementsByClassName("i").length}),r.getById=ut(function(e){return d.appendChild(e).id=b,!n.getElementsByName||!n.getElementsByName(b).length}),r.getById?(o.find.ID=function(e,t){if(typeof t.getElementById!==j&&h){var n=t.getElementById(e);return n&&n.parentNode?[n]:[]}},o.filter.ID=function(e){var t=e.replace(rt,it);return function(e){return e.getAttribute("id")===t}}):(delete o.find.ID,o.filter.ID=function(e){var t=e.replace(rt,it);return function(e){var n=typeof e.getAttributeNode!==j&&e.getAttributeNode("id");return n&&n.value===t}}),o.find.TAG=r.getElementsByTagName?function(e,n){return typeof n.getElementsByTagName!==j?n.getElementsByTagName(e):t}:function(e,t){var n,r=[],i=0,o=t.getElementsByTagName(e);if("*"===e){while(n=o[i++])1===n.nodeType&&r.push(n);return r}return o},o.find.CLASS=r.getElementsByClassName&&function(e,n){return typeof n.getElementsByClassName!==j&&h?n.getElementsByClassName(e):t},m=[],g=[],(r.qsa=K.test(n.querySelectorAll))&&(ut(function(e){e.innerHTML="<select><option selected=''></option></select>",e.querySelectorAll("[selected]").length||g.push("\\["+P+"*(?:value|"+B+")"),e.querySelectorAll(":checked").length||g.push(":checked")}),ut(function(e){var t=n.createElement("input");t.setAttribute("type","hidden"),e.appendChild(t).setAttribute("t",""),e.querySelectorAll("[t^='']").length&&g.push("[*^$]="+P+"*(?:''|\"\")"),e.querySelectorAll(":enabled").length||g.push(":enabled",":disabled"),e.querySelectorAll("*,:x"),g.push(",.*:")})),(r.matchesSelector=K.test(y=d.webkitMatchesSelector||d.mozMatchesSelector||d.oMatchesSelector||d.msMatchesSelector))&&ut(function(e){r.disconnectedMatch=y.call(e,"div"),y.call(e,"[s!='']:x"),m.push("!=",I)}),g=g.length&&RegExp(g.join("|")),m=m.length&&RegExp(m.join("|")),v=K.test(d.contains)||d.compareDocumentPosition?function(e,t){var n=9===e.nodeType?e.documentElement:e,r=t&&t.parentNode;return e===r||!(!r||1!==r.nodeType||!(n.contains?n.contains(r):e.compareDocumentPosition&&16&e.compareDocumentPosition(r)))}:function(e,t){if(t)while(t=t.parentNode)if(t===e)return!0;return!1},A=d.compareDocumentPosition?function(e,t){if(e===t)return S=!0,0;var i=t.compareDocumentPosition&&e.compareDocumentPosition&&e.compareDocumentPosition(t);return i?1&i||!r.sortDetached&&t.compareDocumentPosition(e)===i?e===n||v(w,e)?-1:t===n||v(w,t)?1:c?F.call(c,e)-F.call(c,t):0:4&i?-1:1:e.compareDocumentPosition?-1:1}:function(e,t){var r,i=0,o=e.parentNode,a=t.parentNode,s=[e],l=[t];if(e===t)return S=!0,0;if(!o||!a)return e===n?-1:t===n?1:o?-1:a?1:c?F.call(c,e)-F.call(c,t):0;if(o===a)return pt(e,t);r=e;while(r=r.parentNode)s.unshift(r);r=t;while(r=r.parentNode)l.unshift(r);while(s[i]===l[i])i++;return i?pt(s[i],l[i]):s[i]===w?-1:l[i]===w?1:0},n):f},at.matches=function(e,t){return at(e,null,null,t)},at.matchesSelector=function(e,t){if((e.ownerDocument||e)!==f&&p(e),t=t.replace(Y,"='$1']"),!(!r.matchesSelector||!h||m&&m.test(t)||g&&g.test(t)))try{var n=y.call(e,t);if(n||r.disconnectedMatch||e.document&&11!==e.document.nodeType)return n}catch(i){}return at(t,f,null,[e]).length>0},at.contains=function(e,t){return(e.ownerDocument||e)!==f&&p(e),v(e,t)},at.attr=function(e,n){(e.ownerDocument||e)!==f&&p(e);var i=o.attrHandle[n.toLowerCase()],a=i&&L.call(o.attrHandle,n.toLowerCase())?i(e,n,!h):t;return a===t?r.attributes||!h?e.getAttribute(n):(a=e.getAttributeNode(n))&&a.specified?a.value:null:a},at.error=function(e){throw Error("Syntax error, unrecognized expression: "+e)},at.uniqueSort=function(e){var t,n=[],i=0,o=0;if(S=!r.detectDuplicates,c=!r.sortStable&&e.slice(0),e.sort(A),S){while(t=e[o++])t===e[o]&&(i=n.push(o));while(i--)e.splice(n[i],1)}return e},a=at.getText=function(e){var t,n="",r=0,i=e.nodeType;if(i){if(1===i||9===i||11===i){if("string"==typeof e.textContent)return e.textContent;for(e=e.firstChild;e;e=e.nextSibling)n+=a(e)}else if(3===i||4===i)return e.nodeValue}else for(;t=e[r];r++)n+=a(t);return n},o=at.selectors={cacheLength:50,createPseudo:lt,match:Q,attrHandle:{},find:{},relative:{">":{dir:"parentNode",first:!0}," ":{dir:"parentNode"},"+":{dir:"previousSibling",first:!0},"~":{dir:"previousSibling"}},preFilter:{ATTR:function(e){return e[1]=e[1].replace(rt,it),e[3]=(e[4]||e[5]||"").replace(rt,it),"~="===e[2]&&(e[3]=" "+e[3]+" "),e.slice(0,4)},CHILD:function(e){return e[1]=e[1].toLowerCase(),"nth"===e[1].slice(0,3)?(e[3]||at.error(e[0]),e[4]=+(e[4]?e[5]+(e[6]||1):2*("even"===e[3]||"odd"===e[3])),e[5]=+(e[7]+e[8]||"odd"===e[3])):e[3]&&at.error(e[0]),e},PSEUDO:function(e){var n,r=!e[5]&&e[2];return Q.CHILD.test(e[0])?null:(e[3]&&e[4]!==t?e[2]=e[4]:r&&J.test(r)&&(n=mt(r,!0))&&(n=r.indexOf(")",r.length-n)-r.length)&&(e[0]=e[0].slice(0,n),e[2]=r.slice(0,n)),e.slice(0,3))}},filter:{TAG:function(e){var t=e.replace(rt,it).toLowerCase();return"*"===e?function(){return!0}:function(e){return e.nodeName&&e.nodeName.toLowerCase()===t}},CLASS:function(e){var t=N[e+" "];return t||(t=RegExp("(^|"+P+")"+e+"("+P+"|$)"))&&N(e,function(e){return t.test("string"==typeof e.className&&e.className||typeof e.getAttribute!==j&&e.getAttribute("class")||"")})},ATTR:function(e,t,n){return function(r){var i=at.attr(r,e);return null==i?"!="===t:t?(i+="","="===t?i===n:"!="===t?i!==n:"^="===t?n&&0===i.indexOf(n):"*="===t?n&&i.indexOf(n)>-1:"$="===t?n&&i.slice(-n.length)===n:"~="===t?(" "+i+" ").indexOf(n)>-1:"|="===t?i===n||i.slice(0,n.length+1)===n+"-":!1):!0}},CHILD:function(e,t,n,r,i){var o="nth"!==e.slice(0,3),a="last"!==e.slice(-4),s="of-type"===t;return 1===r&&0===i?function(e){return!!e.parentNode}:function(t,n,l){var u,c,p,f,d,h,g=o!==a?"nextSibling":"previousSibling",m=t.parentNode,y=s&&t.nodeName.toLowerCase(),v=!l&&!s;if(m){if(o){while(g){p=t;while(p=p[g])if(s?p.nodeName.toLowerCase()===y:1===p.nodeType)return!1;h=g="only"===e&&!h&&"nextSibling"}return!0}if(h=[a?m.firstChild:m.lastChild],a&&v){c=m[b]||(m[b]={}),u=c[e]||[],d=u[0]===T&&u[1],f=u[0]===T&&u[2],p=d&&m.childNodes[d];while(p=++d&&p&&p[g]||(f=d=0)||h.pop())if(1===p.nodeType&&++f&&p===t){c[e]=[T,d,f];break}}else if(v&&(u=(t[b]||(t[b]={}))[e])&&u[0]===T)f=u[1];else while(p=++d&&p&&p[g]||(f=d=0)||h.pop())if((s?p.nodeName.toLowerCase()===y:1===p.nodeType)&&++f&&(v&&((p[b]||(p[b]={}))[e]=[T,f]),p===t))break;return f-=i,f===r||0===f%r&&f/r>=0}}},PSEUDO:function(e,t){var n,r=o.pseudos[e]||o.setFilters[e.toLowerCase()]||at.error("unsupported pseudo: "+e);return r[b]?r(t):r.length>1?(n=[e,e,"",t],o.setFilters.hasOwnProperty(e.toLowerCase())?lt(function(e,n){var i,o=r(e,t),a=o.length;while(a--)i=F.call(e,o[a]),e[i]=!(n[i]=o[a])}):function(e){return r(e,0,n)}):r}},pseudos:{not:lt(function(e){var t=[],n=[],r=l(e.replace(z,"$1"));return r[b]?lt(function(e,t,n,i){var o,a=r(e,null,i,[]),s=e.length;while(s--)(o=a[s])&&(e[s]=!(t[s]=o))}):function(e,i,o){return t[0]=e,r(t,null,o,n),!n.pop()}}),has:lt(function(e){return function(t){return at(e,t).length>0}}),contains:lt(function(e){return function(t){return(t.textContent||t.innerText||a(t)).indexOf(e)>-1}}),lang:lt(function(e){return G.test(e||"")||at.error("unsupported lang: "+e),e=e.replace(rt,it).toLowerCase(),function(t){var n;do if(n=h?t.lang:t.getAttribute("xml:lang")||t.getAttribute("lang"))return n=n.toLowerCase(),n===e||0===n.indexOf(e+"-");while((t=t.parentNode)&&1===t.nodeType);return!1}}),target:function(t){var n=e.location&&e.location.hash;return n&&n.slice(1)===t.id},root:function(e){return e===d},focus:function(e){return e===f.activeElement&&(!f.hasFocus||f.hasFocus())&&!!(e.type||e.href||~e.tabIndex)},enabled:function(e){return e.disabled===!1},disabled:function(e){return e.disabled===!0},checked:function(e){var t=e.nodeName.toLowerCase();return"input"===t&&!!e.checked||"option"===t&&!!e.selected},selected:function(e){return e.parentNode&&e.parentNode.selectedIndex,e.selected===!0},empty:function(e){for(e=e.firstChild;e;e=e.nextSibling)if(e.nodeName>"@"||3===e.nodeType||4===e.nodeType)return!1;return!0},parent:function(e){return!o.pseudos.empty(e)},header:function(e){return tt.test(e.nodeName)},input:function(e){return et.test(e.nodeName)},button:function(e){var t=e.nodeName.toLowerCase();return"input"===t&&"button"===e.type||"button"===t},text:function(e){var t;return"input"===e.nodeName.toLowerCase()&&"text"===e.type&&(null==(t=e.getAttribute("type"))||t.toLowerCase()===e.type)},first:ht(function(){return[0]}),last:ht(function(e,t){return[t-1]}),eq:ht(function(e,t,n){return[0>n?n+t:n]}),even:ht(function(e,t){var n=0;for(;t>n;n+=2)e.push(n);return e}),odd:ht(function(e,t){var n=1;for(;t>n;n+=2)e.push(n);return e}),lt:ht(function(e,t,n){var r=0>n?n+t:n;for(;--r>=0;)e.push(r);return e}),gt:ht(function(e,t,n){var r=0>n?n+t:n;for(;t>++r;)e.push(r);return e})}},o.pseudos.nth=o.pseudos.eq;for(n in{radio:!0,checkbox:!0,file:!0,password:!0,image:!0})o.pseudos[n]=ft(n);for(n in{submit:!0,reset:!0})o.pseudos[n]=dt(n);function gt(){}gt.prototype=o.filters=o.pseudos,o.setFilters=new gt;function mt(e,t){var n,r,i,a,s,l,u,c=k[e+" "];if(c)return t?0:c.slice(0);s=e,l=[],u=o.preFilter;while(s){(!n||(r=X.exec(s)))&&(r&&(s=s.slice(r[0].length)||s),l.push(i=[])),n=!1,(r=U.exec(s))&&(n=r.shift(),i.push({value:n,type:r[0].replace(z," ")}),s=s.slice(n.length));for(a in o.filter)!(r=Q[a].exec(s))||u[a]&&!(r=u[a](r))||(n=r.shift(),i.push({value:n,type:a,matches:r}),s=s.slice(n.length));if(!n)break}return t?s.length:s?at.error(e):k(e,l).slice(0)}function yt(e){var t=0,n=e.length,r="";for(;n>t;t++)r+=e[t].value;return r}function vt(e,t,n){var r=t.dir,o=n&&"parentNode"===r,a=C++;return t.first?function(t,n,i){while(t=t[r])if(1===t.nodeType||o)return e(t,n,i)}:function(t,n,s){var l,u,c,p=T+" "+a;if(s){while(t=t[r])if((1===t.nodeType||o)&&e(t,n,s))return!0}else while(t=t[r])if(1===t.nodeType||o)if(c=t[b]||(t[b]={}),(u=c[r])&&u[0]===p){if((l=u[1])===!0||l===i)return l===!0}else if(u=c[r]=[p],u[1]=e(t,n,s)||i,u[1]===!0)return!0}}function bt(e){return e.length>1?function(t,n,r){var i=e.length;while(i--)if(!e[i](t,n,r))return!1;return!0}:e[0]}function xt(e,t,n,r,i){var o,a=[],s=0,l=e.length,u=null!=t;for(;l>s;s++)(o=e[s])&&(!n||n(o,r,i))&&(a.push(o),u&&t.push(s));return a}function wt(e,t,n,r,i,o){return r&&!r[b]&&(r=wt(r)),i&&!i[b]&&(i=wt(i,o)),lt(function(o,a,s,l){var u,c,p,f=[],d=[],h=a.length,g=o||Nt(t||"*",s.nodeType?[s]:s,[]),m=!e||!o&&t?g:xt(g,f,e,s,l),y=n?i||(o?e:h||r)?[]:a:m;if(n&&n(m,y,s,l),r){u=xt(y,d),r(u,[],s,l),c=u.length;while(c--)(p=u[c])&&(y[d[c]]=!(m[d[c]]=p))}if(o){if(i||e){if(i){u=[],c=y.length;while(c--)(p=y[c])&&u.push(m[c]=p);i(null,y=[],u,l)}c=y.length;while(c--)(p=y[c])&&(u=i?F.call(o,p):f[c])>-1&&(o[u]=!(a[u]=p))}}else y=xt(y===a?y.splice(h,y.length):y),i?i(null,a,y,l):M.apply(a,y)})}function Tt(e){var t,n,r,i=e.length,a=o.relative[e[0].type],s=a||o.relative[" "],l=a?1:0,c=vt(function(e){return e===t},s,!0),p=vt(function(e){return F.call(t,e)>-1},s,!0),f=[function(e,n,r){return!a&&(r||n!==u)||((t=n).nodeType?c(e,n,r):p(e,n,r))}];for(;i>l;l++)if(n=o.relative[e[l].type])f=[vt(bt(f),n)];else{if(n=o.filter[e[l].type].apply(null,e[l].matches),n[b]){for(r=++l;i>r;r++)if(o.relative[e[r].type])break;return wt(l>1&&bt(f),l>1&&yt(e.slice(0,l-1).concat({value:" "===e[l-2].type?"*":""})).replace(z,"$1"),n,r>l&&Tt(e.slice(l,r)),i>r&&Tt(e=e.slice(r)),i>r&&yt(e))}f.push(n)}return bt(f)}function Ct(e,t){var n=0,r=t.length>0,a=e.length>0,s=function(s,l,c,p,d){var h,g,m,y=[],v=0,b="0",x=s&&[],w=null!=d,C=u,N=s||a&&o.find.TAG("*",d&&l.parentNode||l),k=T+=null==C?1:Math.random()||.1;for(w&&(u=l!==f&&l,i=n);null!=(h=N[b]);b++){if(a&&h){g=0;while(m=e[g++])if(m(h,l,c)){p.push(h);break}w&&(T=k,i=++n)}r&&((h=!m&&h)&&v--,s&&x.push(h))}if(v+=b,r&&b!==v){g=0;while(m=t[g++])m(x,y,l,c);if(s){if(v>0)while(b--)x[b]||y[b]||(y[b]=q.call(p));y=xt(y)}M.apply(p,y),w&&!s&&y.length>0&&v+t.length>1&&at.uniqueSort(p)}return w&&(T=k,u=C),x};return r?lt(s):s}l=at.compile=function(e,t){var n,r=[],i=[],o=E[e+" "];if(!o){t||(t=mt(e)),n=t.length;while(n--)o=Tt(t[n]),o[b]?r.push(o):i.push(o);o=E(e,Ct(i,r))}return o};function Nt(e,t,n){var r=0,i=t.length;for(;i>r;r++)at(e,t[r],n);return n}function kt(e,t,n,i){var a,s,u,c,p,f=mt(e);if(!i&&1===f.length){if(s=f[0]=f[0].slice(0),s.length>2&&"ID"===(u=s[0]).type&&r.getById&&9===t.nodeType&&h&&o.relative[s[1].type]){if(t=(o.find.ID(u.matches[0].replace(rt,it),t)||[])[0],!t)return n;e=e.slice(s.shift().value.length)}a=Q.needsContext.test(e)?0:s.length;while(a--){if(u=s[a],o.relative[c=u.type])break;if((p=o.find[c])&&(i=p(u.matches[0].replace(rt,it),V.test(s[0].type)&&t.parentNode||t))){if(s.splice(a,1),e=i.length&&yt(s),!e)return M.apply(n,i),n;break}}}return l(e,f)(i,t,!h,n,V.test(e)),n}r.sortStable=b.split("").sort(A).join("")===b,r.detectDuplicates=S,p(),r.sortDetached=ut(function(e){return 1&e.compareDocumentPosition(f.createElement("div"))}),ut(function(e){return e.innerHTML="<a href='#'></a>","#"===e.firstChild.getAttribute("href")})||ct("type|href|height|width",function(e,n,r){return r?t:e.getAttribute(n,"type"===n.toLowerCase()?1:2)}),r.attributes&&ut(function(e){return e.innerHTML="<input/>",e.firstChild.setAttribute("value",""),""===e.firstChild.getAttribute("value")})||ct("value",function(e,n,r){return r||"input"!==e.nodeName.toLowerCase()?t:e.defaultValue}),ut(function(e){return null==e.getAttribute("disabled")})||ct(B,function(e,n,r){var i;return r?t:(i=e.getAttributeNode(n))&&i.specified?i.value:e[n]===!0?n.toLowerCase():null}),x.find=at,x.expr=at.selectors,x.expr[":"]=x.expr.pseudos,x.unique=at.uniqueSort,x.text=at.getText,x.isXMLDoc=at.isXML,x.contains=at.contains}(e);var O={};function F(e){var t=O[e]={};return x.each(e.match(T)||[],function(e,n){t[n]=!0}),t}x.Callbacks=function(e){e="string"==typeof e?O[e]||F(e):x.extend({},e);var n,r,i,o,a,s,l=[],u=!e.once&&[],c=function(t){for(r=e.memory&&t,i=!0,a=s||0,s=0,o=l.length,n=!0;l&&o>a;a++)if(l[a].apply(t[0],t[1])===!1&&e.stopOnFalse){r=!1;break}n=!1,l&&(u?u.length&&c(u.shift()):r?l=[]:p.disable())},p={add:function(){if(l){var t=l.length;(function i(t){x.each(t,function(t,n){var r=x.type(n);"function"===r?e.unique&&p.has(n)||l.push(n):n&&n.length&&"string"!==r&&i(n)})})(arguments),n?o=l.length:r&&(s=t,c(r))}return this},remove:function(){return l&&x.each(arguments,function(e,t){var r;while((r=x.inArray(t,l,r))>-1)l.splice(r,1),n&&(o>=r&&o--,a>=r&&a--)}),this},has:function(e){return e?x.inArray(e,l)>-1:!(!l||!l.length)},empty:function(){return l=[],o=0,this},disable:function(){return l=u=r=t,this},disabled:function(){return!l},lock:function(){return u=t,r||p.disable(),this},locked:function(){return!u},fireWith:function(e,t){return!l||i&&!u||(t=t||[],t=[e,t.slice?t.slice():t],n?u.push(t):c(t)),this},fire:function(){return p.fireWith(this,arguments),this},fired:function(){return!!i}};return p},x.extend({Deferred:function(e){var t=[["resolve","done",x.Callbacks("once memory"),"resolved"],["reject","fail",x.Callbacks("once memory"),"rejected"],["notify","progress",x.Callbacks("memory")]],n="pending",r={state:function(){return n},always:function(){return i.done(arguments).fail(arguments),this},then:function(){var e=arguments;return x.Deferred(function(n){x.each(t,function(t,o){var a=o[0],s=x.isFunction(e[t])&&e[t];i[o[1]](function(){var e=s&&s.apply(this,arguments);e&&x.isFunction(e.promise)?e.promise().done(n.resolve).fail(n.reject).progress(n.notify):n[a+"With"](this===r?n.promise():this,s?[e]:arguments)})}),e=null}).promise()},promise:function(e){return null!=e?x.extend(e,r):r}},i={};return r.pipe=r.then,x.each(t,function(e,o){var a=o[2],s=o[3];r[o[1]]=a.add,s&&a.add(function(){n=s},t[1^e][2].disable,t[2][2].lock),i[o[0]]=function(){return i[o[0]+"With"](this===i?r:this,arguments),this},i[o[0]+"With"]=a.fireWith}),r.promise(i),e&&e.call(i,i),i},when:function(e){var t=0,n=g.call(arguments),r=n.length,i=1!==r||e&&x.isFunction(e.promise)?r:0,o=1===i?e:x.Deferred(),a=function(e,t,n){return function(r){t[e]=this,n[e]=arguments.length>1?g.call(arguments):r,n===s?o.notifyWith(t,n):--i||o.resolveWith(t,n)}},s,l,u;if(r>1)for(s=Array(r),l=Array(r),u=Array(r);r>t;t++)n[t]&&x.isFunction(n[t].promise)?n[t].promise().done(a(t,u,n)).fail(o.reject).progress(a(t,l,s)):--i;return i||o.resolveWith(u,n),o.promise()}}),x.support=function(t){var n,r,o,s,l,u,c,p,f,d=a.createElement("div");if(d.setAttribute("className","t"),d.innerHTML="  <link/><table></table><a href='/a'>a</a><input type='checkbox'/>",n=d.getElementsByTagName("*")||[],r=d.getElementsByTagName("a")[0],!r||!r.style||!n.length)return t;s=a.createElement("select"),u=s.appendChild(a.createElement("option")),o=d.getElementsByTagName("input")[0],r.style.cssText="top:1px;float:left;opacity:.5",t.getSetAttribute="t"!==d.className,t.leadingWhitespace=3===d.firstChild.nodeType,t.tbody=!d.getElementsByTagName("tbody").length,t.htmlSerialize=!!d.getElementsByTagName("link").length,t.style=/top/.test(r.getAttribute("style")),t.hrefNormalized="/a"===r.getAttribute("href"),t.opacity=/^0.5/.test(r.style.opacity),t.cssFloat=!!r.style.cssFloat,t.checkOn=!!o.value,t.optSelected=u.selected,t.enctype=!!a.createElement("form").enctype,t.html5Clone="<:nav></:nav>"!==a.createElement("nav").cloneNode(!0).outerHTML,t.inlineBlockNeedsLayout=!1,t.shrinkWrapBlocks=!1,t.pixelPosition=!1,t.deleteExpando=!0,t.noCloneEvent=!0,t.reliableMarginRight=!0,t.boxSizingReliable=!0,o.checked=!0,t.noCloneChecked=o.cloneNode(!0).checked,s.disabled=!0,t.optDisabled=!u.disabled;try{delete d.test}catch(h){t.deleteExpando=!1}o=a.createElement("input"),o.setAttribute("value",""),t.input=""===o.getAttribute("value"),o.value="t",o.setAttribute("type","radio"),t.radioValue="t"===o.value,o.setAttribute("checked","t"),o.setAttribute("name","t"),l=a.createDocumentFragment(),l.appendChild(o),t.appendChecked=o.checked,t.checkClone=l.cloneNode(!0).cloneNode(!0).lastChild.checked,d.attachEvent&&(d.attachEvent("onclick",function(){t.noCloneEvent=!1}),d.cloneNode(!0).click());for(f in{submit:!0,change:!0,focusin:!0})d.setAttribute(c="on"+f,"t"),t[f+"Bubbles"]=c in e||d.attributes[c].expando===!1;d.style.backgroundClip="content-box",d.cloneNode(!0).style.backgroundClip="",t.clearCloneStyle="content-box"===d.style.backgroundClip;for(f in x(t))break;return t.ownLast="0"!==f,x(function(){var n,r,o,s="padding:0;margin:0;border:0;display:block;box-sizing:content-box;-moz-box-sizing:content-box;-webkit-box-sizing:content-box;",l=a.getElementsByTagName("body")[0];l&&(n=a.createElement("div"),n.style.cssText="border:0;width:0;height:0;position:absolute;top:0;left:-9999px;margin-top:1px",l.appendChild(n).appendChild(d),d.innerHTML="<table><tr><td></td><td>t</td></tr></table>",o=d.getElementsByTagName("td"),o[0].style.cssText="padding:0;margin:0;border:0;display:none",p=0===o[0].offsetHeight,o[0].style.display="",o[1].style.display="none",t.reliableHiddenOffsets=p&&0===o[0].offsetHeight,d.innerHTML="",d.style.cssText="box-sizing:border-box;-moz-box-sizing:border-box;-webkit-box-sizing:border-box;padding:1px;border:1px;display:block;width:4px;margin-top:1%;position:absolute;top:1%;",x.swap(l,null!=l.style.zoom?{zoom:1}:{},function(){t.boxSizing=4===d.offsetWidth}),e.getComputedStyle&&(t.pixelPosition="1%"!==(e.getComputedStyle(d,null)||{}).top,t.boxSizingReliable="4px"===(e.getComputedStyle(d,null)||{width:"4px"}).width,r=d.appendChild(a.createElement("div")),r.style.cssText=d.style.cssText=s,r.style.marginRight=r.style.width="0",d.style.width="1px",t.reliableMarginRight=!parseFloat((e.getComputedStyle(r,null)||{}).marginRight)),typeof d.style.zoom!==i&&(d.innerHTML="",d.style.cssText=s+"width:1px;padding:1px;display:inline;zoom:1",t.inlineBlockNeedsLayout=3===d.offsetWidth,d.style.display="block",d.innerHTML="<div></div>",d.firstChild.style.width="5px",t.shrinkWrapBlocks=3!==d.offsetWidth,t.inlineBlockNeedsLayout&&(l.style.zoom=1)),l.removeChild(n),n=d=o=r=null)}),n=s=l=u=r=o=null,t
-}({});var B=/(?:\{[\s\S]*\}|\[[\s\S]*\])$/,P=/([A-Z])/g;function R(e,n,r,i){if(x.acceptData(e)){var o,a,s=x.expando,l=e.nodeType,u=l?x.cache:e,c=l?e[s]:e[s]&&s;if(c&&u[c]&&(i||u[c].data)||r!==t||"string"!=typeof n)return c||(c=l?e[s]=p.pop()||x.guid++:s),u[c]||(u[c]=l?{}:{toJSON:x.noop}),("object"==typeof n||"function"==typeof n)&&(i?u[c]=x.extend(u[c],n):u[c].data=x.extend(u[c].data,n)),a=u[c],i||(a.data||(a.data={}),a=a.data),r!==t&&(a[x.camelCase(n)]=r),"string"==typeof n?(o=a[n],null==o&&(o=a[x.camelCase(n)])):o=a,o}}function W(e,t,n){if(x.acceptData(e)){var r,i,o=e.nodeType,a=o?x.cache:e,s=o?e[x.expando]:x.expando;if(a[s]){if(t&&(r=n?a[s]:a[s].data)){x.isArray(t)?t=t.concat(x.map(t,x.camelCase)):t in r?t=[t]:(t=x.camelCase(t),t=t in r?[t]:t.split(" ")),i=t.length;while(i--)delete r[t[i]];if(n?!I(r):!x.isEmptyObject(r))return}(n||(delete a[s].data,I(a[s])))&&(o?x.cleanData([e],!0):x.support.deleteExpando||a!=a.window?delete a[s]:a[s]=null)}}}x.extend({cache:{},noData:{applet:!0,embed:!0,object:"clsid:D27CDB6E-AE6D-11cf-96B8-444553540000"},hasData:function(e){return e=e.nodeType?x.cache[e[x.expando]]:e[x.expando],!!e&&!I(e)},data:function(e,t,n){return R(e,t,n)},removeData:function(e,t){return W(e,t)},_data:function(e,t,n){return R(e,t,n,!0)},_removeData:function(e,t){return W(e,t,!0)},acceptData:function(e){if(e.nodeType&&1!==e.nodeType&&9!==e.nodeType)return!1;var t=e.nodeName&&x.noData[e.nodeName.toLowerCase()];return!t||t!==!0&&e.getAttribute("classid")===t}}),x.fn.extend({data:function(e,n){var r,i,o=null,a=0,s=this[0];if(e===t){if(this.length&&(o=x.data(s),1===s.nodeType&&!x._data(s,"parsedAttrs"))){for(r=s.attributes;r.length>a;a++)i=r[a].name,0===i.indexOf("data-")&&(i=x.camelCase(i.slice(5)),$(s,i,o[i]));x._data(s,"parsedAttrs",!0)}return o}return"object"==typeof e?this.each(function(){x.data(this,e)}):arguments.length>1?this.each(function(){x.data(this,e,n)}):s?$(s,e,x.data(s,e)):null},removeData:function(e){return this.each(function(){x.removeData(this,e)})}});function $(e,n,r){if(r===t&&1===e.nodeType){var i="data-"+n.replace(P,"-$1").toLowerCase();if(r=e.getAttribute(i),"string"==typeof r){try{r="true"===r?!0:"false"===r?!1:"null"===r?null:+r+""===r?+r:B.test(r)?x.parseJSON(r):r}catch(o){}x.data(e,n,r)}else r=t}return r}function I(e){var t;for(t in e)if(("data"!==t||!x.isEmptyObject(e[t]))&&"toJSON"!==t)return!1;return!0}x.extend({queue:function(e,n,r){var i;return e?(n=(n||"fx")+"queue",i=x._data(e,n),r&&(!i||x.isArray(r)?i=x._data(e,n,x.makeArray(r)):i.push(r)),i||[]):t},dequeue:function(e,t){t=t||"fx";var n=x.queue(e,t),r=n.length,i=n.shift(),o=x._queueHooks(e,t),a=function(){x.dequeue(e,t)};"inprogress"===i&&(i=n.shift(),r--),i&&("fx"===t&&n.unshift("inprogress"),delete o.stop,i.call(e,a,o)),!r&&o&&o.empty.fire()},_queueHooks:function(e,t){var n=t+"queueHooks";return x._data(e,n)||x._data(e,n,{empty:x.Callbacks("once memory").add(function(){x._removeData(e,t+"queue"),x._removeData(e,n)})})}}),x.fn.extend({queue:function(e,n){var r=2;return"string"!=typeof e&&(n=e,e="fx",r--),r>arguments.length?x.queue(this[0],e):n===t?this:this.each(function(){var t=x.queue(this,e,n);x._queueHooks(this,e),"fx"===e&&"inprogress"!==t[0]&&x.dequeue(this,e)})},dequeue:function(e){return this.each(function(){x.dequeue(this,e)})},delay:function(e,t){return e=x.fx?x.fx.speeds[e]||e:e,t=t||"fx",this.queue(t,function(t,n){var r=setTimeout(t,e);n.stop=function(){clearTimeout(r)}})},clearQueue:function(e){return this.queue(e||"fx",[])},promise:function(e,n){var r,i=1,o=x.Deferred(),a=this,s=this.length,l=function(){--i||o.resolveWith(a,[a])};"string"!=typeof e&&(n=e,e=t),e=e||"fx";while(s--)r=x._data(a[s],e+"queueHooks"),r&&r.empty&&(i++,r.empty.add(l));return l(),o.promise(n)}});var z,X,U=/[\t\r\n\f]/g,V=/\r/g,Y=/^(?:input|select|textarea|button|object)$/i,J=/^(?:a|area)$/i,G=/^(?:checked|selected)$/i,Q=x.support.getSetAttribute,K=x.support.input;x.fn.extend({attr:function(e,t){return x.access(this,x.attr,e,t,arguments.length>1)},removeAttr:function(e){return this.each(function(){x.removeAttr(this,e)})},prop:function(e,t){return x.access(this,x.prop,e,t,arguments.length>1)},removeProp:function(e){return e=x.propFix[e]||e,this.each(function(){try{this[e]=t,delete this[e]}catch(n){}})},addClass:function(e){var t,n,r,i,o,a=0,s=this.length,l="string"==typeof e&&e;if(x.isFunction(e))return this.each(function(t){x(this).addClass(e.call(this,t,this.className))});if(l)for(t=(e||"").match(T)||[];s>a;a++)if(n=this[a],r=1===n.nodeType&&(n.className?(" "+n.className+" ").replace(U," "):" ")){o=0;while(i=t[o++])0>r.indexOf(" "+i+" ")&&(r+=i+" ");n.className=x.trim(r)}return this},removeClass:function(e){var t,n,r,i,o,a=0,s=this.length,l=0===arguments.length||"string"==typeof e&&e;if(x.isFunction(e))return this.each(function(t){x(this).removeClass(e.call(this,t,this.className))});if(l)for(t=(e||"").match(T)||[];s>a;a++)if(n=this[a],r=1===n.nodeType&&(n.className?(" "+n.className+" ").replace(U," "):"")){o=0;while(i=t[o++])while(r.indexOf(" "+i+" ")>=0)r=r.replace(" "+i+" "," ");n.className=e?x.trim(r):""}return this},toggleClass:function(e,t){var n=typeof e;return"boolean"==typeof t&&"string"===n?t?this.addClass(e):this.removeClass(e):x.isFunction(e)?this.each(function(n){x(this).toggleClass(e.call(this,n,this.className,t),t)}):this.each(function(){if("string"===n){var t,r=0,o=x(this),a=e.match(T)||[];while(t=a[r++])o.hasClass(t)?o.removeClass(t):o.addClass(t)}else(n===i||"boolean"===n)&&(this.className&&x._data(this,"__className__",this.className),this.className=this.className||e===!1?"":x._data(this,"__className__")||"")})},hasClass:function(e){var t=" "+e+" ",n=0,r=this.length;for(;r>n;n++)if(1===this[n].nodeType&&(" "+this[n].className+" ").replace(U," ").indexOf(t)>=0)return!0;return!1},val:function(e){var n,r,i,o=this[0];{if(arguments.length)return i=x.isFunction(e),this.each(function(n){var o;1===this.nodeType&&(o=i?e.call(this,n,x(this).val()):e,null==o?o="":"number"==typeof o?o+="":x.isArray(o)&&(o=x.map(o,function(e){return null==e?"":e+""})),r=x.valHooks[this.type]||x.valHooks[this.nodeName.toLowerCase()],r&&"set"in r&&r.set(this,o,"value")!==t||(this.value=o))});if(o)return r=x.valHooks[o.type]||x.valHooks[o.nodeName.toLowerCase()],r&&"get"in r&&(n=r.get(o,"value"))!==t?n:(n=o.value,"string"==typeof n?n.replace(V,""):null==n?"":n)}}}),x.extend({valHooks:{option:{get:function(e){var t=x.find.attr(e,"value");return null!=t?t:e.text}},select:{get:function(e){var t,n,r=e.options,i=e.selectedIndex,o="select-one"===e.type||0>i,a=o?null:[],s=o?i+1:r.length,l=0>i?s:o?i:0;for(;s>l;l++)if(n=r[l],!(!n.selected&&l!==i||(x.support.optDisabled?n.disabled:null!==n.getAttribute("disabled"))||n.parentNode.disabled&&x.nodeName(n.parentNode,"optgroup"))){if(t=x(n).val(),o)return t;a.push(t)}return a},set:function(e,t){var n,r,i=e.options,o=x.makeArray(t),a=i.length;while(a--)r=i[a],(r.selected=x.inArray(x(r).val(),o)>=0)&&(n=!0);return n||(e.selectedIndex=-1),o}}},attr:function(e,n,r){var o,a,s=e.nodeType;if(e&&3!==s&&8!==s&&2!==s)return typeof e.getAttribute===i?x.prop(e,n,r):(1===s&&x.isXMLDoc(e)||(n=n.toLowerCase(),o=x.attrHooks[n]||(x.expr.match.bool.test(n)?X:z)),r===t?o&&"get"in o&&null!==(a=o.get(e,n))?a:(a=x.find.attr(e,n),null==a?t:a):null!==r?o&&"set"in o&&(a=o.set(e,r,n))!==t?a:(e.setAttribute(n,r+""),r):(x.removeAttr(e,n),t))},removeAttr:function(e,t){var n,r,i=0,o=t&&t.match(T);if(o&&1===e.nodeType)while(n=o[i++])r=x.propFix[n]||n,x.expr.match.bool.test(n)?K&&Q||!G.test(n)?e[r]=!1:e[x.camelCase("default-"+n)]=e[r]=!1:x.attr(e,n,""),e.removeAttribute(Q?n:r)},attrHooks:{type:{set:function(e,t){if(!x.support.radioValue&&"radio"===t&&x.nodeName(e,"input")){var n=e.value;return e.setAttribute("type",t),n&&(e.value=n),t}}}},propFix:{"for":"htmlFor","class":"className"},prop:function(e,n,r){var i,o,a,s=e.nodeType;if(e&&3!==s&&8!==s&&2!==s)return a=1!==s||!x.isXMLDoc(e),a&&(n=x.propFix[n]||n,o=x.propHooks[n]),r!==t?o&&"set"in o&&(i=o.set(e,r,n))!==t?i:e[n]=r:o&&"get"in o&&null!==(i=o.get(e,n))?i:e[n]},propHooks:{tabIndex:{get:function(e){var t=x.find.attr(e,"tabindex");return t?parseInt(t,10):Y.test(e.nodeName)||J.test(e.nodeName)&&e.href?0:-1}}}}),X={set:function(e,t,n){return t===!1?x.removeAttr(e,n):K&&Q||!G.test(n)?e.setAttribute(!Q&&x.propFix[n]||n,n):e[x.camelCase("default-"+n)]=e[n]=!0,n}},x.each(x.expr.match.bool.source.match(/\w+/g),function(e,n){var r=x.expr.attrHandle[n]||x.find.attr;x.expr.attrHandle[n]=K&&Q||!G.test(n)?function(e,n,i){var o=x.expr.attrHandle[n],a=i?t:(x.expr.attrHandle[n]=t)!=r(e,n,i)?n.toLowerCase():null;return x.expr.attrHandle[n]=o,a}:function(e,n,r){return r?t:e[x.camelCase("default-"+n)]?n.toLowerCase():null}}),K&&Q||(x.attrHooks.value={set:function(e,n,r){return x.nodeName(e,"input")?(e.defaultValue=n,t):z&&z.set(e,n,r)}}),Q||(z={set:function(e,n,r){var i=e.getAttributeNode(r);return i||e.setAttributeNode(i=e.ownerDocument.createAttribute(r)),i.value=n+="","value"===r||n===e.getAttribute(r)?n:t}},x.expr.attrHandle.id=x.expr.attrHandle.name=x.expr.attrHandle.coords=function(e,n,r){var i;return r?t:(i=e.getAttributeNode(n))&&""!==i.value?i.value:null},x.valHooks.button={get:function(e,n){var r=e.getAttributeNode(n);return r&&r.specified?r.value:t},set:z.set},x.attrHooks.contenteditable={set:function(e,t,n){z.set(e,""===t?!1:t,n)}},x.each(["width","height"],function(e,n){x.attrHooks[n]={set:function(e,r){return""===r?(e.setAttribute(n,"auto"),r):t}}})),x.support.hrefNormalized||x.each(["href","src"],function(e,t){x.propHooks[t]={get:function(e){return e.getAttribute(t,4)}}}),x.support.style||(x.attrHooks.style={get:function(e){return e.style.cssText||t},set:function(e,t){return e.style.cssText=t+""}}),x.support.optSelected||(x.propHooks.selected={get:function(e){var t=e.parentNode;return t&&(t.selectedIndex,t.parentNode&&t.parentNode.selectedIndex),null}}),x.each(["tabIndex","readOnly","maxLength","cellSpacing","cellPadding","rowSpan","colSpan","useMap","frameBorder","contentEditable"],function(){x.propFix[this.toLowerCase()]=this}),x.support.enctype||(x.propFix.enctype="encoding"),x.each(["radio","checkbox"],function(){x.valHooks[this]={set:function(e,n){return x.isArray(n)?e.checked=x.inArray(x(e).val(),n)>=0:t}},x.support.checkOn||(x.valHooks[this].get=function(e){return null===e.getAttribute("value")?"on":e.value})});var Z=/^(?:input|select|textarea)$/i,et=/^key/,tt=/^(?:mouse|contextmenu)|click/,nt=/^(?:focusinfocus|focusoutblur)$/,rt=/^([^.]*)(?:\.(.+)|)$/;function it(){return!0}function ot(){return!1}function at(){try{return a.activeElement}catch(e){}}x.event={global:{},add:function(e,n,r,o,a){var s,l,u,c,p,f,d,h,g,m,y,v=x._data(e);if(v){r.handler&&(c=r,r=c.handler,a=c.selector),r.guid||(r.guid=x.guid++),(l=v.events)||(l=v.events={}),(f=v.handle)||(f=v.handle=function(e){return typeof x===i||e&&x.event.triggered===e.type?t:x.event.dispatch.apply(f.elem,arguments)},f.elem=e),n=(n||"").match(T)||[""],u=n.length;while(u--)s=rt.exec(n[u])||[],g=y=s[1],m=(s[2]||"").split(".").sort(),g&&(p=x.event.special[g]||{},g=(a?p.delegateType:p.bindType)||g,p=x.event.special[g]||{},d=x.extend({type:g,origType:y,data:o,handler:r,guid:r.guid,selector:a,needsContext:a&&x.expr.match.needsContext.test(a),namespace:m.join(".")},c),(h=l[g])||(h=l[g]=[],h.delegateCount=0,p.setup&&p.setup.call(e,o,m,f)!==!1||(e.addEventListener?e.addEventListener(g,f,!1):e.attachEvent&&e.attachEvent("on"+g,f))),p.add&&(p.add.call(e,d),d.handler.guid||(d.handler.guid=r.guid)),a?h.splice(h.delegateCount++,0,d):h.push(d),x.event.global[g]=!0);e=null}},remove:function(e,t,n,r,i){var o,a,s,l,u,c,p,f,d,h,g,m=x.hasData(e)&&x._data(e);if(m&&(c=m.events)){t=(t||"").match(T)||[""],u=t.length;while(u--)if(s=rt.exec(t[u])||[],d=g=s[1],h=(s[2]||"").split(".").sort(),d){p=x.event.special[d]||{},d=(r?p.delegateType:p.bindType)||d,f=c[d]||[],s=s[2]&&RegExp("(^|\\.)"+h.join("\\.(?:.*\\.|)")+"(\\.|$)"),l=o=f.length;while(o--)a=f[o],!i&&g!==a.origType||n&&n.guid!==a.guid||s&&!s.test(a.namespace)||r&&r!==a.selector&&("**"!==r||!a.selector)||(f.splice(o,1),a.selector&&f.delegateCount--,p.remove&&p.remove.call(e,a));l&&!f.length&&(p.teardown&&p.teardown.call(e,h,m.handle)!==!1||x.removeEvent(e,d,m.handle),delete c[d])}else for(d in c)x.event.remove(e,d+t[u],n,r,!0);x.isEmptyObject(c)&&(delete m.handle,x._removeData(e,"events"))}},trigger:function(n,r,i,o){var s,l,u,c,p,f,d,h=[i||a],g=v.call(n,"type")?n.type:n,m=v.call(n,"namespace")?n.namespace.split("."):[];if(u=f=i=i||a,3!==i.nodeType&&8!==i.nodeType&&!nt.test(g+x.event.triggered)&&(g.indexOf(".")>=0&&(m=g.split("."),g=m.shift(),m.sort()),l=0>g.indexOf(":")&&"on"+g,n=n[x.expando]?n:new x.Event(g,"object"==typeof n&&n),n.isTrigger=o?2:3,n.namespace=m.join("."),n.namespace_re=n.namespace?RegExp("(^|\\.)"+m.join("\\.(?:.*\\.|)")+"(\\.|$)"):null,n.result=t,n.target||(n.target=i),r=null==r?[n]:x.makeArray(r,[n]),p=x.event.special[g]||{},o||!p.trigger||p.trigger.apply(i,r)!==!1)){if(!o&&!p.noBubble&&!x.isWindow(i)){for(c=p.delegateType||g,nt.test(c+g)||(u=u.parentNode);u;u=u.parentNode)h.push(u),f=u;f===(i.ownerDocument||a)&&h.push(f.defaultView||f.parentWindow||e)}d=0;while((u=h[d++])&&!n.isPropagationStopped())n.type=d>1?c:p.bindType||g,s=(x._data(u,"events")||{})[n.type]&&x._data(u,"handle"),s&&s.apply(u,r),s=l&&u[l],s&&x.acceptData(u)&&s.apply&&s.apply(u,r)===!1&&n.preventDefault();if(n.type=g,!o&&!n.isDefaultPrevented()&&(!p._default||p._default.apply(h.pop(),r)===!1)&&x.acceptData(i)&&l&&i[g]&&!x.isWindow(i)){f=i[l],f&&(i[l]=null),x.event.triggered=g;try{i[g]()}catch(y){}x.event.triggered=t,f&&(i[l]=f)}return n.result}},dispatch:function(e){e=x.event.fix(e);var n,r,i,o,a,s=[],l=g.call(arguments),u=(x._data(this,"events")||{})[e.type]||[],c=x.event.special[e.type]||{};if(l[0]=e,e.delegateTarget=this,!c.preDispatch||c.preDispatch.call(this,e)!==!1){s=x.event.handlers.call(this,e,u),n=0;while((o=s[n++])&&!e.isPropagationStopped()){e.currentTarget=o.elem,a=0;while((i=o.handlers[a++])&&!e.isImmediatePropagationStopped())(!e.namespace_re||e.namespace_re.test(i.namespace))&&(e.handleObj=i,e.data=i.data,r=((x.event.special[i.origType]||{}).handle||i.handler).apply(o.elem,l),r!==t&&(e.result=r)===!1&&(e.preventDefault(),e.stopPropagation()))}return c.postDispatch&&c.postDispatch.call(this,e),e.result}},handlers:function(e,n){var r,i,o,a,s=[],l=n.delegateCount,u=e.target;if(l&&u.nodeType&&(!e.button||"click"!==e.type))for(;u!=this;u=u.parentNode||this)if(1===u.nodeType&&(u.disabled!==!0||"click"!==e.type)){for(o=[],a=0;l>a;a++)i=n[a],r=i.selector+" ",o[r]===t&&(o[r]=i.needsContext?x(r,this).index(u)>=0:x.find(r,this,null,[u]).length),o[r]&&o.push(i);o.length&&s.push({elem:u,handlers:o})}return n.length>l&&s.push({elem:this,handlers:n.slice(l)}),s},fix:function(e){if(e[x.expando])return e;var t,n,r,i=e.type,o=e,s=this.fixHooks[i];s||(this.fixHooks[i]=s=tt.test(i)?this.mouseHooks:et.test(i)?this.keyHooks:{}),r=s.props?this.props.concat(s.props):this.props,e=new x.Event(o),t=r.length;while(t--)n=r[t],e[n]=o[n];return e.target||(e.target=o.srcElement||a),3===e.target.nodeType&&(e.target=e.target.parentNode),e.metaKey=!!e.metaKey,s.filter?s.filter(e,o):e},props:"altKey bubbles cancelable ctrlKey currentTarget eventPhase metaKey relatedTarget shiftKey target timeStamp view which".split(" "),fixHooks:{},keyHooks:{props:"char charCode key keyCode".split(" "),filter:function(e,t){return null==e.which&&(e.which=null!=t.charCode?t.charCode:t.keyCode),e}},mouseHooks:{props:"button buttons clientX clientY fromElement offsetX offsetY pageX pageY screenX screenY toElement".split(" "),filter:function(e,n){var r,i,o,s=n.button,l=n.fromElement;return null==e.pageX&&null!=n.clientX&&(i=e.target.ownerDocument||a,o=i.documentElement,r=i.body,e.pageX=n.clientX+(o&&o.scrollLeft||r&&r.scrollLeft||0)-(o&&o.clientLeft||r&&r.clientLeft||0),e.pageY=n.clientY+(o&&o.scrollTop||r&&r.scrollTop||0)-(o&&o.clientTop||r&&r.clientTop||0)),!e.relatedTarget&&l&&(e.relatedTarget=l===e.target?n.toElement:l),e.which||s===t||(e.which=1&s?1:2&s?3:4&s?2:0),e}},special:{load:{noBubble:!0},focus:{trigger:function(){if(this!==at()&&this.focus)try{return this.focus(),!1}catch(e){}},delegateType:"focusin"},blur:{trigger:function(){return this===at()&&this.blur?(this.blur(),!1):t},delegateType:"focusout"},click:{trigger:function(){return x.nodeName(this,"input")&&"checkbox"===this.type&&this.click?(this.click(),!1):t},_default:function(e){return x.nodeName(e.target,"a")}},beforeunload:{postDispatch:function(e){e.result!==t&&(e.originalEvent.returnValue=e.result)}}},simulate:function(e,t,n,r){var i=x.extend(new x.Event,n,{type:e,isSimulated:!0,originalEvent:{}});r?x.event.trigger(i,null,t):x.event.dispatch.call(t,i),i.isDefaultPrevented()&&n.preventDefault()}},x.removeEvent=a.removeEventListener?function(e,t,n){e.removeEventListener&&e.removeEventListener(t,n,!1)}:function(e,t,n){var r="on"+t;e.detachEvent&&(typeof e[r]===i&&(e[r]=null),e.detachEvent(r,n))},x.Event=function(e,n){return this instanceof x.Event?(e&&e.type?(this.originalEvent=e,this.type=e.type,this.isDefaultPrevented=e.defaultPrevented||e.returnValue===!1||e.getPreventDefault&&e.getPreventDefault()?it:ot):this.type=e,n&&x.extend(this,n),this.timeStamp=e&&e.timeStamp||x.now(),this[x.expando]=!0,t):new x.Event(e,n)},x.Event.prototype={isDefaultPrevented:ot,isPropagationStopped:ot,isImmediatePropagationStopped:ot,preventDefault:function(){var e=this.originalEvent;this.isDefaultPrevented=it,e&&(e.preventDefault?e.preventDefault():e.returnValue=!1)},stopPropagation:function(){var e=this.originalEvent;this.isPropagationStopped=it,e&&(e.stopPropagation&&e.stopPropagation(),e.cancelBubble=!0)},stopImmediatePropagation:function(){this.isImmediatePropagationStopped=it,this.stopPropagation()}},x.each({mouseenter:"mouseover",mouseleave:"mouseout"},function(e,t){x.event.special[e]={delegateType:t,bindType:t,handle:function(e){var n,r=this,i=e.relatedTarget,o=e.handleObj;return(!i||i!==r&&!x.contains(r,i))&&(e.type=o.origType,n=o.handler.apply(this,arguments),e.type=t),n}}}),x.support.submitBubbles||(x.event.special.submit={setup:function(){return x.nodeName(this,"form")?!1:(x.event.add(this,"click._submit keypress._submit",function(e){var n=e.target,r=x.nodeName(n,"input")||x.nodeName(n,"button")?n.form:t;r&&!x._data(r,"submitBubbles")&&(x.event.add(r,"submit._submit",function(e){e._submit_bubble=!0}),x._data(r,"submitBubbles",!0))}),t)},postDispatch:function(e){e._submit_bubble&&(delete e._submit_bubble,this.parentNode&&!e.isTrigger&&x.event.simulate("submit",this.parentNode,e,!0))},teardown:function(){return x.nodeName(this,"form")?!1:(x.event.remove(this,"._submit"),t)}}),x.support.changeBubbles||(x.event.special.change={setup:function(){return Z.test(this.nodeName)?(("checkbox"===this.type||"radio"===this.type)&&(x.event.add(this,"propertychange._change",function(e){"checked"===e.originalEvent.propertyName&&(this._just_changed=!0)}),x.event.add(this,"click._change",function(e){this._just_changed&&!e.isTrigger&&(this._just_changed=!1),x.event.simulate("change",this,e,!0)})),!1):(x.event.add(this,"beforeactivate._change",function(e){var t=e.target;Z.test(t.nodeName)&&!x._data(t,"changeBubbles")&&(x.event.add(t,"change._change",function(e){!this.parentNode||e.isSimulated||e.isTrigger||x.event.simulate("change",this.parentNode,e,!0)}),x._data(t,"changeBubbles",!0))}),t)},handle:function(e){var n=e.target;return this!==n||e.isSimulated||e.isTrigger||"radio"!==n.type&&"checkbox"!==n.type?e.handleObj.handler.apply(this,arguments):t},teardown:function(){return x.event.remove(this,"._change"),!Z.test(this.nodeName)}}),x.support.focusinBubbles||x.each({focus:"focusin",blur:"focusout"},function(e,t){var n=0,r=function(e){x.event.simulate(t,e.target,x.event.fix(e),!0)};x.event.special[t]={setup:function(){0===n++&&a.addEventListener(e,r,!0)},teardown:function(){0===--n&&a.removeEventListener(e,r,!0)}}}),x.fn.extend({on:function(e,n,r,i,o){var a,s;if("object"==typeof e){"string"!=typeof n&&(r=r||n,n=t);for(a in e)this.on(a,n,r,e[a],o);return this}if(null==r&&null==i?(i=n,r=n=t):null==i&&("string"==typeof n?(i=r,r=t):(i=r,r=n,n=t)),i===!1)i=ot;else if(!i)return this;return 1===o&&(s=i,i=function(e){return x().off(e),s.apply(this,arguments)},i.guid=s.guid||(s.guid=x.guid++)),this.each(function(){x.event.add(this,e,i,r,n)})},one:function(e,t,n,r){return this.on(e,t,n,r,1)},off:function(e,n,r){var i,o;if(e&&e.preventDefault&&e.handleObj)return i=e.handleObj,x(e.delegateTarget).off(i.namespace?i.origType+"."+i.namespace:i.origType,i.selector,i.handler),this;if("object"==typeof e){for(o in e)this.off(o,n,e[o]);return this}return(n===!1||"function"==typeof n)&&(r=n,n=t),r===!1&&(r=ot),this.each(function(){x.event.remove(this,e,r,n)})},trigger:function(e,t){return this.each(function(){x.event.trigger(e,t,this)})},triggerHandler:function(e,n){var r=this[0];return r?x.event.trigger(e,n,r,!0):t}});var st=/^.[^:#\[\.,]*$/,lt=/^(?:parents|prev(?:Until|All))/,ut=x.expr.match.needsContext,ct={children:!0,contents:!0,next:!0,prev:!0};x.fn.extend({find:function(e){var t,n=[],r=this,i=r.length;if("string"!=typeof e)return this.pushStack(x(e).filter(function(){for(t=0;i>t;t++)if(x.contains(r[t],this))return!0}));for(t=0;i>t;t++)x.find(e,r[t],n);return n=this.pushStack(i>1?x.unique(n):n),n.selector=this.selector?this.selector+" "+e:e,n},has:function(e){var t,n=x(e,this),r=n.length;return this.filter(function(){for(t=0;r>t;t++)if(x.contains(this,n[t]))return!0})},not:function(e){return this.pushStack(ft(this,e||[],!0))},filter:function(e){return this.pushStack(ft(this,e||[],!1))},is:function(e){return!!ft(this,"string"==typeof e&&ut.test(e)?x(e):e||[],!1).length},closest:function(e,t){var n,r=0,i=this.length,o=[],a=ut.test(e)||"string"!=typeof e?x(e,t||this.context):0;for(;i>r;r++)for(n=this[r];n&&n!==t;n=n.parentNode)if(11>n.nodeType&&(a?a.index(n)>-1:1===n.nodeType&&x.find.matchesSelector(n,e))){n=o.push(n);break}return this.pushStack(o.length>1?x.unique(o):o)},index:function(e){return e?"string"==typeof e?x.inArray(this[0],x(e)):x.inArray(e.jquery?e[0]:e,this):this[0]&&this[0].parentNode?this.first().prevAll().length:-1},add:function(e,t){var n="string"==typeof e?x(e,t):x.makeArray(e&&e.nodeType?[e]:e),r=x.merge(this.get(),n);return this.pushStack(x.unique(r))},addBack:function(e){return this.add(null==e?this.prevObject:this.prevObject.filter(e))}});function pt(e,t){do e=e[t];while(e&&1!==e.nodeType);return e}x.each({parent:function(e){var t=e.parentNode;return t&&11!==t.nodeType?t:null},parents:function(e){return x.dir(e,"parentNode")},parentsUntil:function(e,t,n){return x.dir(e,"parentNode",n)},next:function(e){return pt(e,"nextSibling")},prev:function(e){return pt(e,"previousSibling")},nextAll:function(e){return x.dir(e,"nextSibling")},prevAll:function(e){return x.dir(e,"previousSibling")},nextUntil:function(e,t,n){return x.dir(e,"nextSibling",n)},prevUntil:function(e,t,n){return x.dir(e,"previousSibling",n)},siblings:function(e){return x.sibling((e.parentNode||{}).firstChild,e)},children:function(e){return x.sibling(e.firstChild)},contents:function(e){return x.nodeName(e,"iframe")?e.contentDocument||e.contentWindow.document:x.merge([],e.childNodes)}},function(e,t){x.fn[e]=function(n,r){var i=x.map(this,t,n);return"Until"!==e.slice(-5)&&(r=n),r&&"string"==typeof r&&(i=x.filter(r,i)),this.length>1&&(ct[e]||(i=x.unique(i)),lt.test(e)&&(i=i.reverse())),this.pushStack(i)}}),x.extend({filter:function(e,t,n){var r=t[0];return n&&(e=":not("+e+")"),1===t.length&&1===r.nodeType?x.find.matchesSelector(r,e)?[r]:[]:x.find.matches(e,x.grep(t,function(e){return 1===e.nodeType}))},dir:function(e,n,r){var i=[],o=e[n];while(o&&9!==o.nodeType&&(r===t||1!==o.nodeType||!x(o).is(r)))1===o.nodeType&&i.push(o),o=o[n];return i},sibling:function(e,t){var n=[];for(;e;e=e.nextSibling)1===e.nodeType&&e!==t&&n.push(e);return n}});function ft(e,t,n){if(x.isFunction(t))return x.grep(e,function(e,r){return!!t.call(e,r,e)!==n});if(t.nodeType)return x.grep(e,function(e){return e===t!==n});if("string"==typeof t){if(st.test(t))return x.filter(t,e,n);t=x.filter(t,e)}return x.grep(e,function(e){return x.inArray(e,t)>=0!==n})}function dt(e){var t=ht.split("|"),n=e.createDocumentFragment();if(n.createElement)while(t.length)n.createElement(t.pop());return n}var ht="abbr|article|aside|audio|bdi|canvas|data|datalist|details|figcaption|figure|footer|header|hgroup|mark|meter|nav|output|progress|section|summary|time|video",gt=/ jQuery\d+="(?:null|\d+)"/g,mt=RegExp("<(?:"+ht+")[\\s/>]","i"),yt=/^\s+/,vt=/<(?!area|br|col|embed|hr|img|input|link|meta|param)(([\w:]+)[^>]*)\/>/gi,bt=/<([\w:]+)/,xt=/<tbody/i,wt=/<|&#?\w+;/,Tt=/<(?:script|style|link)/i,Ct=/^(?:checkbox|radio)$/i,Nt=/checked\s*(?:[^=]|=\s*.checked.)/i,kt=/^$|\/(?:java|ecma)script/i,Et=/^true\/(.*)/,St=/^\s*<!(?:\[CDATA\[|--)|(?:\]\]|--)>\s*$/g,At={option:[1,"<select multiple='multiple'>","</select>"],legend:[1,"<fieldset>","</fieldset>"],area:[1,"<map>","</map>"],param:[1,"<object>","</object>"],thead:[1,"<table>","</table>"],tr:[2,"<table><tbody>","</tbody></table>"],col:[2,"<table><tbody></tbody><colgroup>","</colgroup></table>"],td:[3,"<table><tbody><tr>","</tr></tbody></table>"],_default:x.support.htmlSerialize?[0,"",""]:[1,"X<div>","</div>"]},jt=dt(a),Dt=jt.appendChild(a.createElement("div"));At.optgroup=At.option,At.tbody=At.tfoot=At.colgroup=At.caption=At.thead,At.th=At.td,x.fn.extend({text:function(e){return x.access(this,function(e){return e===t?x.text(this):this.empty().append((this[0]&&this[0].ownerDocument||a).createTextNode(e))},null,e,arguments.length)},append:function(){return this.domManip(arguments,function(e){if(1===this.nodeType||11===this.nodeType||9===this.nodeType){var t=Lt(this,e);t.appendChild(e)}})},prepend:function(){return this.domManip(arguments,function(e){if(1===this.nodeType||11===this.nodeType||9===this.nodeType){var t=Lt(this,e);t.insertBefore(e,t.firstChild)}})},before:function(){return this.domManip(arguments,function(e){this.parentNode&&this.parentNode.insertBefore(e,this)})},after:function(){return this.domManip(arguments,function(e){this.parentNode&&this.parentNode.insertBefore(e,this.nextSibling)})},remove:function(e,t){var n,r=e?x.filter(e,this):this,i=0;for(;null!=(n=r[i]);i++)t||1!==n.nodeType||x.cleanData(Ft(n)),n.parentNode&&(t&&x.contains(n.ownerDocument,n)&&_t(Ft(n,"script")),n.parentNode.removeChild(n));return this},empty:function(){var e,t=0;for(;null!=(e=this[t]);t++){1===e.nodeType&&x.cleanData(Ft(e,!1));while(e.firstChild)e.removeChild(e.firstChild);e.options&&x.nodeName(e,"select")&&(e.options.length=0)}return this},clone:function(e,t){return e=null==e?!1:e,t=null==t?e:t,this.map(function(){return x.clone(this,e,t)})},html:function(e){return x.access(this,function(e){var n=this[0]||{},r=0,i=this.length;if(e===t)return 1===n.nodeType?n.innerHTML.replace(gt,""):t;if(!("string"!=typeof e||Tt.test(e)||!x.support.htmlSerialize&&mt.test(e)||!x.support.leadingWhitespace&&yt.test(e)||At[(bt.exec(e)||["",""])[1].toLowerCase()])){e=e.replace(vt,"<$1></$2>");try{for(;i>r;r++)n=this[r]||{},1===n.nodeType&&(x.cleanData(Ft(n,!1)),n.innerHTML=e);n=0}catch(o){}}n&&this.empty().append(e)},null,e,arguments.length)},replaceWith:function(){var e=x.map(this,function(e){return[e.nextSibling,e.parentNode]}),t=0;return this.domManip(arguments,function(n){var r=e[t++],i=e[t++];i&&(r&&r.parentNode!==i&&(r=this.nextSibling),x(this).remove(),i.insertBefore(n,r))},!0),t?this:this.remove()},detach:function(e){return this.remove(e,!0)},domManip:function(e,t,n){e=d.apply([],e);var r,i,o,a,s,l,u=0,c=this.length,p=this,f=c-1,h=e[0],g=x.isFunction(h);if(g||!(1>=c||"string"!=typeof h||x.support.checkClone)&&Nt.test(h))return this.each(function(r){var i=p.eq(r);g&&(e[0]=h.call(this,r,i.html())),i.domManip(e,t,n)});if(c&&(l=x.buildFragment(e,this[0].ownerDocument,!1,!n&&this),r=l.firstChild,1===l.childNodes.length&&(l=r),r)){for(a=x.map(Ft(l,"script"),Ht),o=a.length;c>u;u++)i=l,u!==f&&(i=x.clone(i,!0,!0),o&&x.merge(a,Ft(i,"script"))),t.call(this[u],i,u);if(o)for(s=a[a.length-1].ownerDocument,x.map(a,qt),u=0;o>u;u++)i=a[u],kt.test(i.type||"")&&!x._data(i,"globalEval")&&x.contains(s,i)&&(i.src?x._evalUrl(i.src):x.globalEval((i.text||i.textContent||i.innerHTML||"").replace(St,"")));l=r=null}return this}});function Lt(e,t){return x.nodeName(e,"table")&&x.nodeName(1===t.nodeType?t:t.firstChild,"tr")?e.getElementsByTagName("tbody")[0]||e.appendChild(e.ownerDocument.createElement("tbody")):e}function Ht(e){return e.type=(null!==x.find.attr(e,"type"))+"/"+e.type,e}function qt(e){var t=Et.exec(e.type);return t?e.type=t[1]:e.removeAttribute("type"),e}function _t(e,t){var n,r=0;for(;null!=(n=e[r]);r++)x._data(n,"globalEval",!t||x._data(t[r],"globalEval"))}function Mt(e,t){if(1===t.nodeType&&x.hasData(e)){var n,r,i,o=x._data(e),a=x._data(t,o),s=o.events;if(s){delete a.handle,a.events={};for(n in s)for(r=0,i=s[n].length;i>r;r++)x.event.add(t,n,s[n][r])}a.data&&(a.data=x.extend({},a.data))}}function Ot(e,t){var n,r,i;if(1===t.nodeType){if(n=t.nodeName.toLowerCase(),!x.support.noCloneEvent&&t[x.expando]){i=x._data(t);for(r in i.events)x.removeEvent(t,r,i.handle);t.removeAttribute(x.expando)}"script"===n&&t.text!==e.text?(Ht(t).text=e.text,qt(t)):"object"===n?(t.parentNode&&(t.outerHTML=e.outerHTML),x.support.html5Clone&&e.innerHTML&&!x.trim(t.innerHTML)&&(t.innerHTML=e.innerHTML)):"input"===n&&Ct.test(e.type)?(t.defaultChecked=t.checked=e.checked,t.value!==e.value&&(t.value=e.value)):"option"===n?t.defaultSelected=t.selected=e.defaultSelected:("input"===n||"textarea"===n)&&(t.defaultValue=e.defaultValue)}}x.each({appendTo:"append",prependTo:"prepend",insertBefore:"before",insertAfter:"after",replaceAll:"replaceWith"},function(e,t){x.fn[e]=function(e){var n,r=0,i=[],o=x(e),a=o.length-1;for(;a>=r;r++)n=r===a?this:this.clone(!0),x(o[r])[t](n),h.apply(i,n.get());return this.pushStack(i)}});function Ft(e,n){var r,o,a=0,s=typeof e.getElementsByTagName!==i?e.getElementsByTagName(n||"*"):typeof e.querySelectorAll!==i?e.querySelectorAll(n||"*"):t;if(!s)for(s=[],r=e.childNodes||e;null!=(o=r[a]);a++)!n||x.nodeName(o,n)?s.push(o):x.merge(s,Ft(o,n));return n===t||n&&x.nodeName(e,n)?x.merge([e],s):s}function Bt(e){Ct.test(e.type)&&(e.defaultChecked=e.checked)}x.extend({clone:function(e,t,n){var r,i,o,a,s,l=x.contains(e.ownerDocument,e);if(x.support.html5Clone||x.isXMLDoc(e)||!mt.test("<"+e.nodeName+">")?o=e.cloneNode(!0):(Dt.innerHTML=e.outerHTML,Dt.removeChild(o=Dt.firstChild)),!(x.support.noCloneEvent&&x.support.noCloneChecked||1!==e.nodeType&&11!==e.nodeType||x.isXMLDoc(e)))for(r=Ft(o),s=Ft(e),a=0;null!=(i=s[a]);++a)r[a]&&Ot(i,r[a]);if(t)if(n)for(s=s||Ft(e),r=r||Ft(o),a=0;null!=(i=s[a]);a++)Mt(i,r[a]);else Mt(e,o);return r=Ft(o,"script"),r.length>0&&_t(r,!l&&Ft(e,"script")),r=s=i=null,o},buildFragment:function(e,t,n,r){var i,o,a,s,l,u,c,p=e.length,f=dt(t),d=[],h=0;for(;p>h;h++)if(o=e[h],o||0===o)if("object"===x.type(o))x.merge(d,o.nodeType?[o]:o);else if(wt.test(o)){s=s||f.appendChild(t.createElement("div")),l=(bt.exec(o)||["",""])[1].toLowerCase(),c=At[l]||At._default,s.innerHTML=c[1]+o.replace(vt,"<$1></$2>")+c[2],i=c[0];while(i--)s=s.lastChild;if(!x.support.leadingWhitespace&&yt.test(o)&&d.push(t.createTextNode(yt.exec(o)[0])),!x.support.tbody){o="table"!==l||xt.test(o)?"<table>"!==c[1]||xt.test(o)?0:s:s.firstChild,i=o&&o.childNodes.length;while(i--)x.nodeName(u=o.childNodes[i],"tbody")&&!u.childNodes.length&&o.removeChild(u)}x.merge(d,s.childNodes),s.textContent="";while(s.firstChild)s.removeChild(s.firstChild);s=f.lastChild}else d.push(t.createTextNode(o));s&&f.removeChild(s),x.support.appendChecked||x.grep(Ft(d,"input"),Bt),h=0;while(o=d[h++])if((!r||-1===x.inArray(o,r))&&(a=x.contains(o.ownerDocument,o),s=Ft(f.appendChild(o),"script"),a&&_t(s),n)){i=0;while(o=s[i++])kt.test(o.type||"")&&n.push(o)}return s=null,f},cleanData:function(e,t){var n,r,o,a,s=0,l=x.expando,u=x.cache,c=x.support.deleteExpando,f=x.event.special;for(;null!=(n=e[s]);s++)if((t||x.acceptData(n))&&(o=n[l],a=o&&u[o])){if(a.events)for(r in a.events)f[r]?x.event.remove(n,r):x.removeEvent(n,r,a.handle);
-u[o]&&(delete u[o],c?delete n[l]:typeof n.removeAttribute!==i?n.removeAttribute(l):n[l]=null,p.push(o))}},_evalUrl:function(e){return x.ajax({url:e,type:"GET",dataType:"script",async:!1,global:!1,"throws":!0})}}),x.fn.extend({wrapAll:function(e){if(x.isFunction(e))return this.each(function(t){x(this).wrapAll(e.call(this,t))});if(this[0]){var t=x(e,this[0].ownerDocument).eq(0).clone(!0);this[0].parentNode&&t.insertBefore(this[0]),t.map(function(){var e=this;while(e.firstChild&&1===e.firstChild.nodeType)e=e.firstChild;return e}).append(this)}return this},wrapInner:function(e){return x.isFunction(e)?this.each(function(t){x(this).wrapInner(e.call(this,t))}):this.each(function(){var t=x(this),n=t.contents();n.length?n.wrapAll(e):t.append(e)})},wrap:function(e){var t=x.isFunction(e);return this.each(function(n){x(this).wrapAll(t?e.call(this,n):e)})},unwrap:function(){return this.parent().each(function(){x.nodeName(this,"body")||x(this).replaceWith(this.childNodes)}).end()}});var Pt,Rt,Wt,$t=/alpha\([^)]*\)/i,It=/opacity\s*=\s*([^)]*)/,zt=/^(top|right|bottom|left)$/,Xt=/^(none|table(?!-c[ea]).+)/,Ut=/^margin/,Vt=RegExp("^("+w+")(.*)$","i"),Yt=RegExp("^("+w+")(?!px)[a-z%]+$","i"),Jt=RegExp("^([+-])=("+w+")","i"),Gt={BODY:"block"},Qt={position:"absolute",visibility:"hidden",display:"block"},Kt={letterSpacing:0,fontWeight:400},Zt=["Top","Right","Bottom","Left"],en=["Webkit","O","Moz","ms"];function tn(e,t){if(t in e)return t;var n=t.charAt(0).toUpperCase()+t.slice(1),r=t,i=en.length;while(i--)if(t=en[i]+n,t in e)return t;return r}function nn(e,t){return e=t||e,"none"===x.css(e,"display")||!x.contains(e.ownerDocument,e)}function rn(e,t){var n,r,i,o=[],a=0,s=e.length;for(;s>a;a++)r=e[a],r.style&&(o[a]=x._data(r,"olddisplay"),n=r.style.display,t?(o[a]||"none"!==n||(r.style.display=""),""===r.style.display&&nn(r)&&(o[a]=x._data(r,"olddisplay",ln(r.nodeName)))):o[a]||(i=nn(r),(n&&"none"!==n||!i)&&x._data(r,"olddisplay",i?n:x.css(r,"display"))));for(a=0;s>a;a++)r=e[a],r.style&&(t&&"none"!==r.style.display&&""!==r.style.display||(r.style.display=t?o[a]||"":"none"));return e}x.fn.extend({css:function(e,n){return x.access(this,function(e,n,r){var i,o,a={},s=0;if(x.isArray(n)){for(o=Rt(e),i=n.length;i>s;s++)a[n[s]]=x.css(e,n[s],!1,o);return a}return r!==t?x.style(e,n,r):x.css(e,n)},e,n,arguments.length>1)},show:function(){return rn(this,!0)},hide:function(){return rn(this)},toggle:function(e){return"boolean"==typeof e?e?this.show():this.hide():this.each(function(){nn(this)?x(this).show():x(this).hide()})}}),x.extend({cssHooks:{opacity:{get:function(e,t){if(t){var n=Wt(e,"opacity");return""===n?"1":n}}}},cssNumber:{columnCount:!0,fillOpacity:!0,fontWeight:!0,lineHeight:!0,opacity:!0,order:!0,orphans:!0,widows:!0,zIndex:!0,zoom:!0},cssProps:{"float":x.support.cssFloat?"cssFloat":"styleFloat"},style:function(e,n,r,i){if(e&&3!==e.nodeType&&8!==e.nodeType&&e.style){var o,a,s,l=x.camelCase(n),u=e.style;if(n=x.cssProps[l]||(x.cssProps[l]=tn(u,l)),s=x.cssHooks[n]||x.cssHooks[l],r===t)return s&&"get"in s&&(o=s.get(e,!1,i))!==t?o:u[n];if(a=typeof r,"string"===a&&(o=Jt.exec(r))&&(r=(o[1]+1)*o[2]+parseFloat(x.css(e,n)),a="number"),!(null==r||"number"===a&&isNaN(r)||("number"!==a||x.cssNumber[l]||(r+="px"),x.support.clearCloneStyle||""!==r||0!==n.indexOf("background")||(u[n]="inherit"),s&&"set"in s&&(r=s.set(e,r,i))===t)))try{u[n]=r}catch(c){}}},css:function(e,n,r,i){var o,a,s,l=x.camelCase(n);return n=x.cssProps[l]||(x.cssProps[l]=tn(e.style,l)),s=x.cssHooks[n]||x.cssHooks[l],s&&"get"in s&&(a=s.get(e,!0,r)),a===t&&(a=Wt(e,n,i)),"normal"===a&&n in Kt&&(a=Kt[n]),""===r||r?(o=parseFloat(a),r===!0||x.isNumeric(o)?o||0:a):a}}),e.getComputedStyle?(Rt=function(t){return e.getComputedStyle(t,null)},Wt=function(e,n,r){var i,o,a,s=r||Rt(e),l=s?s.getPropertyValue(n)||s[n]:t,u=e.style;return s&&(""!==l||x.contains(e.ownerDocument,e)||(l=x.style(e,n)),Yt.test(l)&&Ut.test(n)&&(i=u.width,o=u.minWidth,a=u.maxWidth,u.minWidth=u.maxWidth=u.width=l,l=s.width,u.width=i,u.minWidth=o,u.maxWidth=a)),l}):a.documentElement.currentStyle&&(Rt=function(e){return e.currentStyle},Wt=function(e,n,r){var i,o,a,s=r||Rt(e),l=s?s[n]:t,u=e.style;return null==l&&u&&u[n]&&(l=u[n]),Yt.test(l)&&!zt.test(n)&&(i=u.left,o=e.runtimeStyle,a=o&&o.left,a&&(o.left=e.currentStyle.left),u.left="fontSize"===n?"1em":l,l=u.pixelLeft+"px",u.left=i,a&&(o.left=a)),""===l?"auto":l});function on(e,t,n){var r=Vt.exec(t);return r?Math.max(0,r[1]-(n||0))+(r[2]||"px"):t}function an(e,t,n,r,i){var o=n===(r?"border":"content")?4:"width"===t?1:0,a=0;for(;4>o;o+=2)"margin"===n&&(a+=x.css(e,n+Zt[o],!0,i)),r?("content"===n&&(a-=x.css(e,"padding"+Zt[o],!0,i)),"margin"!==n&&(a-=x.css(e,"border"+Zt[o]+"Width",!0,i))):(a+=x.css(e,"padding"+Zt[o],!0,i),"padding"!==n&&(a+=x.css(e,"border"+Zt[o]+"Width",!0,i)));return a}function sn(e,t,n){var r=!0,i="width"===t?e.offsetWidth:e.offsetHeight,o=Rt(e),a=x.support.boxSizing&&"border-box"===x.css(e,"boxSizing",!1,o);if(0>=i||null==i){if(i=Wt(e,t,o),(0>i||null==i)&&(i=e.style[t]),Yt.test(i))return i;r=a&&(x.support.boxSizingReliable||i===e.style[t]),i=parseFloat(i)||0}return i+an(e,t,n||(a?"border":"content"),r,o)+"px"}function ln(e){var t=a,n=Gt[e];return n||(n=un(e,t),"none"!==n&&n||(Pt=(Pt||x("<iframe frameborder='0' width='0' height='0'/>").css("cssText","display:block !important")).appendTo(t.documentElement),t=(Pt[0].contentWindow||Pt[0].contentDocument).document,t.write("<!doctype html><html><body>"),t.close(),n=un(e,t),Pt.detach()),Gt[e]=n),n}function un(e,t){var n=x(t.createElement(e)).appendTo(t.body),r=x.css(n[0],"display");return n.remove(),r}x.each(["height","width"],function(e,n){x.cssHooks[n]={get:function(e,r,i){return r?0===e.offsetWidth&&Xt.test(x.css(e,"display"))?x.swap(e,Qt,function(){return sn(e,n,i)}):sn(e,n,i):t},set:function(e,t,r){var i=r&&Rt(e);return on(e,t,r?an(e,n,r,x.support.boxSizing&&"border-box"===x.css(e,"boxSizing",!1,i),i):0)}}}),x.support.opacity||(x.cssHooks.opacity={get:function(e,t){return It.test((t&&e.currentStyle?e.currentStyle.filter:e.style.filter)||"")?.01*parseFloat(RegExp.$1)+"":t?"1":""},set:function(e,t){var n=e.style,r=e.currentStyle,i=x.isNumeric(t)?"alpha(opacity="+100*t+")":"",o=r&&r.filter||n.filter||"";n.zoom=1,(t>=1||""===t)&&""===x.trim(o.replace($t,""))&&n.removeAttribute&&(n.removeAttribute("filter"),""===t||r&&!r.filter)||(n.filter=$t.test(o)?o.replace($t,i):o+" "+i)}}),x(function(){x.support.reliableMarginRight||(x.cssHooks.marginRight={get:function(e,n){return n?x.swap(e,{display:"inline-block"},Wt,[e,"marginRight"]):t}}),!x.support.pixelPosition&&x.fn.position&&x.each(["top","left"],function(e,n){x.cssHooks[n]={get:function(e,r){return r?(r=Wt(e,n),Yt.test(r)?x(e).position()[n]+"px":r):t}}})}),x.expr&&x.expr.filters&&(x.expr.filters.hidden=function(e){return 0>=e.offsetWidth&&0>=e.offsetHeight||!x.support.reliableHiddenOffsets&&"none"===(e.style&&e.style.display||x.css(e,"display"))},x.expr.filters.visible=function(e){return!x.expr.filters.hidden(e)}),x.each({margin:"",padding:"",border:"Width"},function(e,t){x.cssHooks[e+t]={expand:function(n){var r=0,i={},o="string"==typeof n?n.split(" "):[n];for(;4>r;r++)i[e+Zt[r]+t]=o[r]||o[r-2]||o[0];return i}},Ut.test(e)||(x.cssHooks[e+t].set=on)});var cn=/%20/g,pn=/\[\]$/,fn=/\r?\n/g,dn=/^(?:submit|button|image|reset|file)$/i,hn=/^(?:input|select|textarea|keygen)/i;x.fn.extend({serialize:function(){return x.param(this.serializeArray())},serializeArray:function(){return this.map(function(){var e=x.prop(this,"elements");return e?x.makeArray(e):this}).filter(function(){var e=this.type;return this.name&&!x(this).is(":disabled")&&hn.test(this.nodeName)&&!dn.test(e)&&(this.checked||!Ct.test(e))}).map(function(e,t){var n=x(this).val();return null==n?null:x.isArray(n)?x.map(n,function(e){return{name:t.name,value:e.replace(fn,"\r\n")}}):{name:t.name,value:n.replace(fn,"\r\n")}}).get()}}),x.param=function(e,n){var r,i=[],o=function(e,t){t=x.isFunction(t)?t():null==t?"":t,i[i.length]=encodeURIComponent(e)+"="+encodeURIComponent(t)};if(n===t&&(n=x.ajaxSettings&&x.ajaxSettings.traditional),x.isArray(e)||e.jquery&&!x.isPlainObject(e))x.each(e,function(){o(this.name,this.value)});else for(r in e)gn(r,e[r],n,o);return i.join("&").replace(cn,"+")};function gn(e,t,n,r){var i;if(x.isArray(t))x.each(t,function(t,i){n||pn.test(e)?r(e,i):gn(e+"["+("object"==typeof i?t:"")+"]",i,n,r)});else if(n||"object"!==x.type(t))r(e,t);else for(i in t)gn(e+"["+i+"]",t[i],n,r)}x.each("blur focus focusin focusout load resize scroll unload click dblclick mousedown mouseup mousemove mouseover mouseout mouseenter mouseleave change select submit keydown keypress keyup error contextmenu".split(" "),function(e,t){x.fn[t]=function(e,n){return arguments.length>0?this.on(t,null,e,n):this.trigger(t)}}),x.fn.extend({hover:function(e,t){return this.mouseenter(e).mouseleave(t||e)},bind:function(e,t,n){return this.on(e,null,t,n)},unbind:function(e,t){return this.off(e,null,t)},delegate:function(e,t,n,r){return this.on(t,e,n,r)},undelegate:function(e,t,n){return 1===arguments.length?this.off(e,"**"):this.off(t,e||"**",n)}});var mn,yn,vn=x.now(),bn=/\?/,xn=/#.*$/,wn=/([?&])_=[^&]*/,Tn=/^(.*?):[ \t]*([^\r\n]*)\r?$/gm,Cn=/^(?:about|app|app-storage|.+-extension|file|res|widget):$/,Nn=/^(?:GET|HEAD)$/,kn=/^\/\//,En=/^([\w.+-]+:)(?:\/\/([^\/?#:]*)(?::(\d+)|)|)/,Sn=x.fn.load,An={},jn={},Dn="*/".concat("*");try{yn=o.href}catch(Ln){yn=a.createElement("a"),yn.href="",yn=yn.href}mn=En.exec(yn.toLowerCase())||[];function Hn(e){return function(t,n){"string"!=typeof t&&(n=t,t="*");var r,i=0,o=t.toLowerCase().match(T)||[];if(x.isFunction(n))while(r=o[i++])"+"===r[0]?(r=r.slice(1)||"*",(e[r]=e[r]||[]).unshift(n)):(e[r]=e[r]||[]).push(n)}}function qn(e,n,r,i){var o={},a=e===jn;function s(l){var u;return o[l]=!0,x.each(e[l]||[],function(e,l){var c=l(n,r,i);return"string"!=typeof c||a||o[c]?a?!(u=c):t:(n.dataTypes.unshift(c),s(c),!1)}),u}return s(n.dataTypes[0])||!o["*"]&&s("*")}function _n(e,n){var r,i,o=x.ajaxSettings.flatOptions||{};for(i in n)n[i]!==t&&((o[i]?e:r||(r={}))[i]=n[i]);return r&&x.extend(!0,e,r),e}x.fn.load=function(e,n,r){if("string"!=typeof e&&Sn)return Sn.apply(this,arguments);var i,o,a,s=this,l=e.indexOf(" ");return l>=0&&(i=e.slice(l,e.length),e=e.slice(0,l)),x.isFunction(n)?(r=n,n=t):n&&"object"==typeof n&&(a="POST"),s.length>0&&x.ajax({url:e,type:a,dataType:"html",data:n}).done(function(e){o=arguments,s.html(i?x("<div>").append(x.parseHTML(e)).find(i):e)}).complete(r&&function(e,t){s.each(r,o||[e.responseText,t,e])}),this},x.each(["ajaxStart","ajaxStop","ajaxComplete","ajaxError","ajaxSuccess","ajaxSend"],function(e,t){x.fn[t]=function(e){return this.on(t,e)}}),x.extend({active:0,lastModified:{},etag:{},ajaxSettings:{url:yn,type:"GET",isLocal:Cn.test(mn[1]),global:!0,processData:!0,async:!0,contentType:"application/x-www-form-urlencoded; charset=UTF-8",accepts:{"*":Dn,text:"text/plain",html:"text/html",xml:"application/xml, text/xml",json:"application/json, text/javascript"},contents:{xml:/xml/,html:/html/,json:/json/},responseFields:{xml:"responseXML",text:"responseText",json:"responseJSON"},converters:{"* text":String,"text html":!0,"text json":x.parseJSON,"text xml":x.parseXML},flatOptions:{url:!0,context:!0}},ajaxSetup:function(e,t){return t?_n(_n(e,x.ajaxSettings),t):_n(x.ajaxSettings,e)},ajaxPrefilter:Hn(An),ajaxTransport:Hn(jn),ajax:function(e,n){"object"==typeof e&&(n=e,e=t),n=n||{};var r,i,o,a,s,l,u,c,p=x.ajaxSetup({},n),f=p.context||p,d=p.context&&(f.nodeType||f.jquery)?x(f):x.event,h=x.Deferred(),g=x.Callbacks("once memory"),m=p.statusCode||{},y={},v={},b=0,w="canceled",C={readyState:0,getResponseHeader:function(e){var t;if(2===b){if(!c){c={};while(t=Tn.exec(a))c[t[1].toLowerCase()]=t[2]}t=c[e.toLowerCase()]}return null==t?null:t},getAllResponseHeaders:function(){return 2===b?a:null},setRequestHeader:function(e,t){var n=e.toLowerCase();return b||(e=v[n]=v[n]||e,y[e]=t),this},overrideMimeType:function(e){return b||(p.mimeType=e),this},statusCode:function(e){var t;if(e)if(2>b)for(t in e)m[t]=[m[t],e[t]];else C.always(e[C.status]);return this},abort:function(e){var t=e||w;return u&&u.abort(t),k(0,t),this}};if(h.promise(C).complete=g.add,C.success=C.done,C.error=C.fail,p.url=((e||p.url||yn)+"").replace(xn,"").replace(kn,mn[1]+"//"),p.type=n.method||n.type||p.method||p.type,p.dataTypes=x.trim(p.dataType||"*").toLowerCase().match(T)||[""],null==p.crossDomain&&(r=En.exec(p.url.toLowerCase()),p.crossDomain=!(!r||r[1]===mn[1]&&r[2]===mn[2]&&(r[3]||("http:"===r[1]?"80":"443"))===(mn[3]||("http:"===mn[1]?"80":"443")))),p.data&&p.processData&&"string"!=typeof p.data&&(p.data=x.param(p.data,p.traditional)),qn(An,p,n,C),2===b)return C;l=p.global,l&&0===x.active++&&x.event.trigger("ajaxStart"),p.type=p.type.toUpperCase(),p.hasContent=!Nn.test(p.type),o=p.url,p.hasContent||(p.data&&(o=p.url+=(bn.test(o)?"&":"?")+p.data,delete p.data),p.cache===!1&&(p.url=wn.test(o)?o.replace(wn,"$1_="+vn++):o+(bn.test(o)?"&":"?")+"_="+vn++)),p.ifModified&&(x.lastModified[o]&&C.setRequestHeader("If-Modified-Since",x.lastModified[o]),x.etag[o]&&C.setRequestHeader("If-None-Match",x.etag[o])),(p.data&&p.hasContent&&p.contentType!==!1||n.contentType)&&C.setRequestHeader("Content-Type",p.contentType),C.setRequestHeader("Accept",p.dataTypes[0]&&p.accepts[p.dataTypes[0]]?p.accepts[p.dataTypes[0]]+("*"!==p.dataTypes[0]?", "+Dn+"; q=0.01":""):p.accepts["*"]);for(i in p.headers)C.setRequestHeader(i,p.headers[i]);if(p.beforeSend&&(p.beforeSend.call(f,C,p)===!1||2===b))return C.abort();w="abort";for(i in{success:1,error:1,complete:1})C[i](p[i]);if(u=qn(jn,p,n,C)){C.readyState=1,l&&d.trigger("ajaxSend",[C,p]),p.async&&p.timeout>0&&(s=setTimeout(function(){C.abort("timeout")},p.timeout));try{b=1,u.send(y,k)}catch(N){if(!(2>b))throw N;k(-1,N)}}else k(-1,"No Transport");function k(e,n,r,i){var c,y,v,w,T,N=n;2!==b&&(b=2,s&&clearTimeout(s),u=t,a=i||"",C.readyState=e>0?4:0,c=e>=200&&300>e||304===e,r&&(w=Mn(p,C,r)),w=On(p,w,C,c),c?(p.ifModified&&(T=C.getResponseHeader("Last-Modified"),T&&(x.lastModified[o]=T),T=C.getResponseHeader("etag"),T&&(x.etag[o]=T)),204===e||"HEAD"===p.type?N="nocontent":304===e?N="notmodified":(N=w.state,y=w.data,v=w.error,c=!v)):(v=N,(e||!N)&&(N="error",0>e&&(e=0))),C.status=e,C.statusText=(n||N)+"",c?h.resolveWith(f,[y,N,C]):h.rejectWith(f,[C,N,v]),C.statusCode(m),m=t,l&&d.trigger(c?"ajaxSuccess":"ajaxError",[C,p,c?y:v]),g.fireWith(f,[C,N]),l&&(d.trigger("ajaxComplete",[C,p]),--x.active||x.event.trigger("ajaxStop")))}return C},getJSON:function(e,t,n){return x.get(e,t,n,"json")},getScript:function(e,n){return x.get(e,t,n,"script")}}),x.each(["get","post"],function(e,n){x[n]=function(e,r,i,o){return x.isFunction(r)&&(o=o||i,i=r,r=t),x.ajax({url:e,type:n,dataType:o,data:r,success:i})}});function Mn(e,n,r){var i,o,a,s,l=e.contents,u=e.dataTypes;while("*"===u[0])u.shift(),o===t&&(o=e.mimeType||n.getResponseHeader("Content-Type"));if(o)for(s in l)if(l[s]&&l[s].test(o)){u.unshift(s);break}if(u[0]in r)a=u[0];else{for(s in r){if(!u[0]||e.converters[s+" "+u[0]]){a=s;break}i||(i=s)}a=a||i}return a?(a!==u[0]&&u.unshift(a),r[a]):t}function On(e,t,n,r){var i,o,a,s,l,u={},c=e.dataTypes.slice();if(c[1])for(a in e.converters)u[a.toLowerCase()]=e.converters[a];o=c.shift();while(o)if(e.responseFields[o]&&(n[e.responseFields[o]]=t),!l&&r&&e.dataFilter&&(t=e.dataFilter(t,e.dataType)),l=o,o=c.shift())if("*"===o)o=l;else if("*"!==l&&l!==o){if(a=u[l+" "+o]||u["* "+o],!a)for(i in u)if(s=i.split(" "),s[1]===o&&(a=u[l+" "+s[0]]||u["* "+s[0]])){a===!0?a=u[i]:u[i]!==!0&&(o=s[0],c.unshift(s[1]));break}if(a!==!0)if(a&&e["throws"])t=a(t);else try{t=a(t)}catch(p){return{state:"parsererror",error:a?p:"No conversion from "+l+" to "+o}}}return{state:"success",data:t}}x.ajaxSetup({accepts:{script:"text/javascript, application/javascript, application/ecmascript, application/x-ecmascript"},contents:{script:/(?:java|ecma)script/},converters:{"text script":function(e){return x.globalEval(e),e}}}),x.ajaxPrefilter("script",function(e){e.cache===t&&(e.cache=!1),e.crossDomain&&(e.type="GET",e.global=!1)}),x.ajaxTransport("script",function(e){if(e.crossDomain){var n,r=a.head||x("head")[0]||a.documentElement;return{send:function(t,i){n=a.createElement("script"),n.async=!0,e.scriptCharset&&(n.charset=e.scriptCharset),n.src=e.url,n.onload=n.onreadystatechange=function(e,t){(t||!n.readyState||/loaded|complete/.test(n.readyState))&&(n.onload=n.onreadystatechange=null,n.parentNode&&n.parentNode.removeChild(n),n=null,t||i(200,"success"))},r.insertBefore(n,r.firstChild)},abort:function(){n&&n.onload(t,!0)}}}});var Fn=[],Bn=/(=)\?(?=&|$)|\?\?/;x.ajaxSetup({jsonp:"callback",jsonpCallback:function(){var e=Fn.pop()||x.expando+"_"+vn++;return this[e]=!0,e}}),x.ajaxPrefilter("json jsonp",function(n,r,i){var o,a,s,l=n.jsonp!==!1&&(Bn.test(n.url)?"url":"string"==typeof n.data&&!(n.contentType||"").indexOf("application/x-www-form-urlencoded")&&Bn.test(n.data)&&"data");return l||"jsonp"===n.dataTypes[0]?(o=n.jsonpCallback=x.isFunction(n.jsonpCallback)?n.jsonpCallback():n.jsonpCallback,l?n[l]=n[l].replace(Bn,"$1"+o):n.jsonp!==!1&&(n.url+=(bn.test(n.url)?"&":"?")+n.jsonp+"="+o),n.converters["script json"]=function(){return s||x.error(o+" was not called"),s[0]},n.dataTypes[0]="json",a=e[o],e[o]=function(){s=arguments},i.always(function(){e[o]=a,n[o]&&(n.jsonpCallback=r.jsonpCallback,Fn.push(o)),s&&x.isFunction(a)&&a(s[0]),s=a=t}),"script"):t});var Pn,Rn,Wn=0,$n=e.ActiveXObject&&function(){var e;for(e in Pn)Pn[e](t,!0)};function In(){try{return new e.XMLHttpRequest}catch(t){}}function zn(){try{return new e.ActiveXObject("Microsoft.XMLHTTP")}catch(t){}}x.ajaxSettings.xhr=e.ActiveXObject?function(){return!this.isLocal&&In()||zn()}:In,Rn=x.ajaxSettings.xhr(),x.support.cors=!!Rn&&"withCredentials"in Rn,Rn=x.support.ajax=!!Rn,Rn&&x.ajaxTransport(function(n){if(!n.crossDomain||x.support.cors){var r;return{send:function(i,o){var a,s,l=n.xhr();if(n.username?l.open(n.type,n.url,n.async,n.username,n.password):l.open(n.type,n.url,n.async),n.xhrFields)for(s in n.xhrFields)l[s]=n.xhrFields[s];n.mimeType&&l.overrideMimeType&&l.overrideMimeType(n.mimeType),n.crossDomain||i["X-Requested-With"]||(i["X-Requested-With"]="XMLHttpRequest");try{for(s in i)l.setRequestHeader(s,i[s])}catch(u){}l.send(n.hasContent&&n.data||null),r=function(e,i){var s,u,c,p;try{if(r&&(i||4===l.readyState))if(r=t,a&&(l.onreadystatechange=x.noop,$n&&delete Pn[a]),i)4!==l.readyState&&l.abort();else{p={},s=l.status,u=l.getAllResponseHeaders(),"string"==typeof l.responseText&&(p.text=l.responseText);try{c=l.statusText}catch(f){c=""}s||!n.isLocal||n.crossDomain?1223===s&&(s=204):s=p.text?200:404}}catch(d){i||o(-1,d)}p&&o(s,c,p,u)},n.async?4===l.readyState?setTimeout(r):(a=++Wn,$n&&(Pn||(Pn={},x(e).unload($n)),Pn[a]=r),l.onreadystatechange=r):r()},abort:function(){r&&r(t,!0)}}}});var Xn,Un,Vn=/^(?:toggle|show|hide)$/,Yn=RegExp("^(?:([+-])=|)("+w+")([a-z%]*)$","i"),Jn=/queueHooks$/,Gn=[nr],Qn={"*":[function(e,t){var n=this.createTween(e,t),r=n.cur(),i=Yn.exec(t),o=i&&i[3]||(x.cssNumber[e]?"":"px"),a=(x.cssNumber[e]||"px"!==o&&+r)&&Yn.exec(x.css(n.elem,e)),s=1,l=20;if(a&&a[3]!==o){o=o||a[3],i=i||[],a=+r||1;do s=s||".5",a/=s,x.style(n.elem,e,a+o);while(s!==(s=n.cur()/r)&&1!==s&&--l)}return i&&(a=n.start=+a||+r||0,n.unit=o,n.end=i[1]?a+(i[1]+1)*i[2]:+i[2]),n}]};function Kn(){return setTimeout(function(){Xn=t}),Xn=x.now()}function Zn(e,t,n){var r,i=(Qn[t]||[]).concat(Qn["*"]),o=0,a=i.length;for(;a>o;o++)if(r=i[o].call(n,t,e))return r}function er(e,t,n){var r,i,o=0,a=Gn.length,s=x.Deferred().always(function(){delete l.elem}),l=function(){if(i)return!1;var t=Xn||Kn(),n=Math.max(0,u.startTime+u.duration-t),r=n/u.duration||0,o=1-r,a=0,l=u.tweens.length;for(;l>a;a++)u.tweens[a].run(o);return s.notifyWith(e,[u,o,n]),1>o&&l?n:(s.resolveWith(e,[u]),!1)},u=s.promise({elem:e,props:x.extend({},t),opts:x.extend(!0,{specialEasing:{}},n),originalProperties:t,originalOptions:n,startTime:Xn||Kn(),duration:n.duration,tweens:[],createTween:function(t,n){var r=x.Tween(e,u.opts,t,n,u.opts.specialEasing[t]||u.opts.easing);return u.tweens.push(r),r},stop:function(t){var n=0,r=t?u.tweens.length:0;if(i)return this;for(i=!0;r>n;n++)u.tweens[n].run(1);return t?s.resolveWith(e,[u,t]):s.rejectWith(e,[u,t]),this}}),c=u.props;for(tr(c,u.opts.specialEasing);a>o;o++)if(r=Gn[o].call(u,e,c,u.opts))return r;return x.map(c,Zn,u),x.isFunction(u.opts.start)&&u.opts.start.call(e,u),x.fx.timer(x.extend(l,{elem:e,anim:u,queue:u.opts.queue})),u.progress(u.opts.progress).done(u.opts.done,u.opts.complete).fail(u.opts.fail).always(u.opts.always)}function tr(e,t){var n,r,i,o,a;for(n in e)if(r=x.camelCase(n),i=t[r],o=e[n],x.isArray(o)&&(i=o[1],o=e[n]=o[0]),n!==r&&(e[r]=o,delete e[n]),a=x.cssHooks[r],a&&"expand"in a){o=a.expand(o),delete e[r];for(n in o)n in e||(e[n]=o[n],t[n]=i)}else t[r]=i}x.Animation=x.extend(er,{tweener:function(e,t){x.isFunction(e)?(t=e,e=["*"]):e=e.split(" ");var n,r=0,i=e.length;for(;i>r;r++)n=e[r],Qn[n]=Qn[n]||[],Qn[n].unshift(t)},prefilter:function(e,t){t?Gn.unshift(e):Gn.push(e)}});function nr(e,t,n){var r,i,o,a,s,l,u=this,c={},p=e.style,f=e.nodeType&&nn(e),d=x._data(e,"fxshow");n.queue||(s=x._queueHooks(e,"fx"),null==s.unqueued&&(s.unqueued=0,l=s.empty.fire,s.empty.fire=function(){s.unqueued||l()}),s.unqueued++,u.always(function(){u.always(function(){s.unqueued--,x.queue(e,"fx").length||s.empty.fire()})})),1===e.nodeType&&("height"in t||"width"in t)&&(n.overflow=[p.overflow,p.overflowX,p.overflowY],"inline"===x.css(e,"display")&&"none"===x.css(e,"float")&&(x.support.inlineBlockNeedsLayout&&"inline"!==ln(e.nodeName)?p.zoom=1:p.display="inline-block")),n.overflow&&(p.overflow="hidden",x.support.shrinkWrapBlocks||u.always(function(){p.overflow=n.overflow[0],p.overflowX=n.overflow[1],p.overflowY=n.overflow[2]}));for(r in t)if(i=t[r],Vn.exec(i)){if(delete t[r],o=o||"toggle"===i,i===(f?"hide":"show"))continue;c[r]=d&&d[r]||x.style(e,r)}if(!x.isEmptyObject(c)){d?"hidden"in d&&(f=d.hidden):d=x._data(e,"fxshow",{}),o&&(d.hidden=!f),f?x(e).show():u.done(function(){x(e).hide()}),u.done(function(){var t;x._removeData(e,"fxshow");for(t in c)x.style(e,t,c[t])});for(r in c)a=Zn(f?d[r]:0,r,u),r in d||(d[r]=a.start,f&&(a.end=a.start,a.start="width"===r||"height"===r?1:0))}}function rr(e,t,n,r,i){return new rr.prototype.init(e,t,n,r,i)}x.Tween=rr,rr.prototype={constructor:rr,init:function(e,t,n,r,i,o){this.elem=e,this.prop=n,this.easing=i||"swing",this.options=t,this.start=this.now=this.cur(),this.end=r,this.unit=o||(x.cssNumber[n]?"":"px")},cur:function(){var e=rr.propHooks[this.prop];return e&&e.get?e.get(this):rr.propHooks._default.get(this)},run:function(e){var t,n=rr.propHooks[this.prop];return this.pos=t=this.options.duration?x.easing[this.easing](e,this.options.duration*e,0,1,this.options.duration):e,this.now=(this.end-this.start)*t+this.start,this.options.step&&this.options.step.call(this.elem,this.now,this),n&&n.set?n.set(this):rr.propHooks._default.set(this),this}},rr.prototype.init.prototype=rr.prototype,rr.propHooks={_default:{get:function(e){var t;return null==e.elem[e.prop]||e.elem.style&&null!=e.elem.style[e.prop]?(t=x.css(e.elem,e.prop,""),t&&"auto"!==t?t:0):e.elem[e.prop]},set:function(e){x.fx.step[e.prop]?x.fx.step[e.prop](e):e.elem.style&&(null!=e.elem.style[x.cssProps[e.prop]]||x.cssHooks[e.prop])?x.style(e.elem,e.prop,e.now+e.unit):e.elem[e.prop]=e.now}}},rr.propHooks.scrollTop=rr.propHooks.scrollLeft={set:function(e){e.elem.nodeType&&e.elem.parentNode&&(e.elem[e.prop]=e.now)}},x.each(["toggle","show","hide"],function(e,t){var n=x.fn[t];x.fn[t]=function(e,r,i){return null==e||"boolean"==typeof e?n.apply(this,arguments):this.animate(ir(t,!0),e,r,i)}}),x.fn.extend({fadeTo:function(e,t,n,r){return this.filter(nn).css("opacity",0).show().end().animate({opacity:t},e,n,r)},animate:function(e,t,n,r){var i=x.isEmptyObject(e),o=x.speed(t,n,r),a=function(){var t=er(this,x.extend({},e),o);(i||x._data(this,"finish"))&&t.stop(!0)};return a.finish=a,i||o.queue===!1?this.each(a):this.queue(o.queue,a)},stop:function(e,n,r){var i=function(e){var t=e.stop;delete e.stop,t(r)};return"string"!=typeof e&&(r=n,n=e,e=t),n&&e!==!1&&this.queue(e||"fx",[]),this.each(function(){var t=!0,n=null!=e&&e+"queueHooks",o=x.timers,a=x._data(this);if(n)a[n]&&a[n].stop&&i(a[n]);else for(n in a)a[n]&&a[n].stop&&Jn.test(n)&&i(a[n]);for(n=o.length;n--;)o[n].elem!==this||null!=e&&o[n].queue!==e||(o[n].anim.stop(r),t=!1,o.splice(n,1));(t||!r)&&x.dequeue(this,e)})},finish:function(e){return e!==!1&&(e=e||"fx"),this.each(function(){var t,n=x._data(this),r=n[e+"queue"],i=n[e+"queueHooks"],o=x.timers,a=r?r.length:0;for(n.finish=!0,x.queue(this,e,[]),i&&i.stop&&i.stop.call(this,!0),t=o.length;t--;)o[t].elem===this&&o[t].queue===e&&(o[t].anim.stop(!0),o.splice(t,1));for(t=0;a>t;t++)r[t]&&r[t].finish&&r[t].finish.call(this);delete n.finish})}});function ir(e,t){var n,r={height:e},i=0;for(t=t?1:0;4>i;i+=2-t)n=Zt[i],r["margin"+n]=r["padding"+n]=e;return t&&(r.opacity=r.width=e),r}x.each({slideDown:ir("show"),slideUp:ir("hide"),slideToggle:ir("toggle"),fadeIn:{opacity:"show"},fadeOut:{opacity:"hide"},fadeToggle:{opacity:"toggle"}},function(e,t){x.fn[e]=function(e,n,r){return this.animate(t,e,n,r)}}),x.speed=function(e,t,n){var r=e&&"object"==typeof e?x.extend({},e):{complete:n||!n&&t||x.isFunction(e)&&e,duration:e,easing:n&&t||t&&!x.isFunction(t)&&t};return r.duration=x.fx.off?0:"number"==typeof r.duration?r.duration:r.duration in x.fx.speeds?x.fx.speeds[r.duration]:x.fx.speeds._default,(null==r.queue||r.queue===!0)&&(r.queue="fx"),r.old=r.complete,r.complete=function(){x.isFunction(r.old)&&r.old.call(this),r.queue&&x.dequeue(this,r.queue)},r},x.easing={linear:function(e){return e},swing:function(e){return.5-Math.cos(e*Math.PI)/2}},x.timers=[],x.fx=rr.prototype.init,x.fx.tick=function(){var e,n=x.timers,r=0;for(Xn=x.now();n.length>r;r++)e=n[r],e()||n[r]!==e||n.splice(r--,1);n.length||x.fx.stop(),Xn=t},x.fx.timer=function(e){e()&&x.timers.push(e)&&x.fx.start()},x.fx.interval=13,x.fx.start=function(){Un||(Un=setInterval(x.fx.tick,x.fx.interval))},x.fx.stop=function(){clearInterval(Un),Un=null},x.fx.speeds={slow:600,fast:200,_default:400},x.fx.step={},x.expr&&x.expr.filters&&(x.expr.filters.animated=function(e){return x.grep(x.timers,function(t){return e===t.elem}).length}),x.fn.offset=function(e){if(arguments.length)return e===t?this:this.each(function(t){x.offset.setOffset(this,e,t)});var n,r,o={top:0,left:0},a=this[0],s=a&&a.ownerDocument;if(s)return n=s.documentElement,x.contains(n,a)?(typeof a.getBoundingClientRect!==i&&(o=a.getBoundingClientRect()),r=or(s),{top:o.top+(r.pageYOffset||n.scrollTop)-(n.clientTop||0),left:o.left+(r.pageXOffset||n.scrollLeft)-(n.clientLeft||0)}):o},x.offset={setOffset:function(e,t,n){var r=x.css(e,"position");"static"===r&&(e.style.position="relative");var i=x(e),o=i.offset(),a=x.css(e,"top"),s=x.css(e,"left"),l=("absolute"===r||"fixed"===r)&&x.inArray("auto",[a,s])>-1,u={},c={},p,f;l?(c=i.position(),p=c.top,f=c.left):(p=parseFloat(a)||0,f=parseFloat(s)||0),x.isFunction(t)&&(t=t.call(e,n,o)),null!=t.top&&(u.top=t.top-o.top+p),null!=t.left&&(u.left=t.left-o.left+f),"using"in t?t.using.call(e,u):i.css(u)}},x.fn.extend({position:function(){if(this[0]){var e,t,n={top:0,left:0},r=this[0];return"fixed"===x.css(r,"position")?t=r.getBoundingClientRect():(e=this.offsetParent(),t=this.offset(),x.nodeName(e[0],"html")||(n=e.offset()),n.top+=x.css(e[0],"borderTopWidth",!0),n.left+=x.css(e[0],"borderLeftWidth",!0)),{top:t.top-n.top-x.css(r,"marginTop",!0),left:t.left-n.left-x.css(r,"marginLeft",!0)}}},offsetParent:function(){return this.map(function(){var e=this.offsetParent||s;while(e&&!x.nodeName(e,"html")&&"static"===x.css(e,"position"))e=e.offsetParent;return e||s})}}),x.each({scrollLeft:"pageXOffset",scrollTop:"pageYOffset"},function(e,n){var r=/Y/.test(n);x.fn[e]=function(i){return x.access(this,function(e,i,o){var a=or(e);return o===t?a?n in a?a[n]:a.document.documentElement[i]:e[i]:(a?a.scrollTo(r?x(a).scrollLeft():o,r?o:x(a).scrollTop()):e[i]=o,t)},e,i,arguments.length,null)}});function or(e){return x.isWindow(e)?e:9===e.nodeType?e.defaultView||e.parentWindow:!1}x.each({Height:"height",Width:"width"},function(e,n){x.each({padding:"inner"+e,content:n,"":"outer"+e},function(r,i){x.fn[i]=function(i,o){var a=arguments.length&&(r||"boolean"!=typeof i),s=r||(i===!0||o===!0?"margin":"border");return x.access(this,function(n,r,i){var o;return x.isWindow(n)?n.document.documentElement["client"+e]:9===n.nodeType?(o=n.documentElement,Math.max(n.body["scroll"+e],o["scroll"+e],n.body["offset"+e],o["offset"+e],o["client"+e])):i===t?x.css(n,r,s):x.style(n,r,i,s)},n,a?i:t,a,null)}})}),x.fn.size=function(){return this.length},x.fn.andSelf=x.fn.addBack,"object"==typeof module&&module&&"object"==typeof module.exports?module.exports=x:(e.jQuery=e.$=x,"function"==typeof define&&define.amd&&define("jquery",[],function(){return x}))})(window);
diff --git a/js/jquery-3.6.0.min.js b/js/jquery-3.6.0.min.js
new file mode 100644
index 000000000..c4c6022f2
--- /dev/null
+++ b/js/jquery-3.6.0.min.js
@@ -0,0 +1,2 @@
+/*! jQuery v3.6.0 | (c) OpenJS Foundation and other contributors | jquery.org/license */
+!function(e,t){"use strict";"object"==typeof module&&"object"==typeof module.exports?module.exports=e.document?t(e,!0):function(e){if(!e.document)throw new Error("jQuery requires a window with a document");return t(e)}:t(e)}("undefined"!=typeof window?window:this,function(C,e){"use strict";var t=[],r=Object.getPrototypeOf,s=t.slice,g=t.flat?function(e){return t.flat.call(e)}:function(e){return t.concat.apply([],e)},u=t.push,i=t.indexOf,n={},o=n.toString,v=n.hasOwnProperty,a=v.toString,l=a.call(Object),y={},m=function(e){return"function"==typeof e&&"number"!=typeof e.nodeType&&"function"!=typeof e.item},x=function(e){return null!=e&&e===e.window},E=C.document,c={type:!0,src:!0,nonce:!0,noModule:!0};function b(e,t,n){var r,i,o=(n=n||E).createElement("script");if(o.text=e,t)for(r in c)(i=t[r]||t.getAttribute&&t.getAttribute(r))&&o.setAttribute(r,i);n.head.appendChild(o).parentNode.removeChild(o)}function w(e){return null==e?e+"":"object"==typeof e||"function"==typeof e?n[o.call(e)]||"object":typeof e}var f="3.6.0",S=function(e,t){return new S.fn.init(e,t)};function p(e){var t=!!e&&"length"in e&&e.length,n=w(e);return!m(e)&&!x(e)&&("array"===n||0===t||"number"==typeof t&&0<t&&t-1 in e)}S.fn=S.prototype={jquery:f,constructor:S,length:0,toArray:function(){return s.call(this)},get:function(e){return null==e?s.call(this):e<0?this[e+this.length]:this[e]},pushStack:function(e){var t=S.merge(this.constructor(),e);return t.prevObject=this,t},each:function(e){return S.each(this,e)},map:function(n){return this.pushStack(S.map(this,function(e,t){return n.call(e,t,e)}))},slice:function(){return this.pushStack(s.apply(this,arguments))},first:function(){return this.eq(0)},last:function(){return this.eq(-1)},even:function(){return this.pushStack(S.grep(this,function(e,t){return(t+1)%2}))},odd:function(){return this.pushStack(S.grep(this,function(e,t){return t%2}))},eq:function(e){var t=this.length,n=+e+(e<0?t:0);return this.pushStack(0<=n&&n<t?[this[n]]:[])},end:function(){return this.prevObject||this.constructor()},push:u,sort:t.sort,splice:t.splice},S.extend=S.fn.extend=function(){var e,t,n,r,i,o,a=arguments[0]||{},s=1,u=arguments.length,l=!1;for("boolean"==typeof a&&(l=a,a=arguments[s]||{},s++),"object"==typeof a||m(a)||(a={}),s===u&&(a=this,s--);s<u;s++)if(null!=(e=arguments[s]))for(t in e)r=e[t],"__proto__"!==t&&a!==r&&(l&&r&&(S.isPlainObject(r)||(i=Array.isArray(r)))?(n=a[t],o=i&&!Array.isArray(n)?[]:i||S.isPlainObject(n)?n:{},i=!1,a[t]=S.extend(l,o,r)):void 0!==r&&(a[t]=r));return a},S.extend({expando:"jQuery"+(f+Math.random()).replace(/\D/g,""),isReady:!0,error:function(e){throw new Error(e)},noop:function(){},isPlainObject:function(e){var t,n;return!(!e||"[object Object]"!==o.call(e))&&(!(t=r(e))||"function"==typeof(n=v.call(t,"constructor")&&t.constructor)&&a.call(n)===l)},isEmptyObject:function(e){var t;for(t in e)return!1;return!0},globalEval:function(e,t,n){b(e,{nonce:t&&t.nonce},n)},each:function(e,t){var n,r=0;if(p(e)){for(n=e.length;r<n;r++)if(!1===t.call(e[r],r,e[r]))break}else for(r in e)if(!1===t.call(e[r],r,e[r]))break;return e},makeArray:function(e,t){var n=t||[];return null!=e&&(p(Object(e))?S.merge(n,"string"==typeof e?[e]:e):u.call(n,e)),n},inArray:function(e,t,n){return null==t?-1:i.call(t,e,n)},merge:function(e,t){for(var n=+t.length,r=0,i=e.length;r<n;r++)e[i++]=t[r];return e.length=i,e},grep:function(e,t,n){for(var r=[],i=0,o=e.length,a=!n;i<o;i++)!t(e[i],i)!==a&&r.push(e[i]);return r},map:function(e,t,n){var r,i,o=0,a=[];if(p(e))for(r=e.length;o<r;o++)null!=(i=t(e[o],o,n))&&a.push(i);else for(o in e)null!=(i=t(e[o],o,n))&&a.push(i);return g(a)},guid:1,support:y}),"function"==typeof Symbol&&(S.fn[Symbol.iterator]=t[Symbol.iterator]),S.each("Boolean Number String Function Array Date RegExp Object Error Symbol".split(" "),function(e,t){n["[object "+t+"]"]=t.toLowerCase()});var d=function(n){var e,d,b,o,i,h,f,g,w,u,l,T,C,a,E,v,s,c,y,S="sizzle"+1*new Date,p=n.document,k=0,r=0,m=ue(),x=ue(),A=ue(),N=ue(),j=function(e,t){return e===t&&(l=!0),0},D={}.hasOwnProperty,t=[],q=t.pop,L=t.push,H=t.push,O=t.slice,P=function(e,t){for(var n=0,r=e.length;n<r;n++)if(e[n]===t)return n;return-1},R="checked|selected|async|autofocus|autoplay|controls|defer|disabled|hidden|ismap|loop|multiple|open|readonly|required|scoped",M="[\\x20\\t\\r\\n\\f]",I="(?:\\\\[\\da-fA-F]{1,6}"+M+"?|\\\\[^\\r\\n\\f]|[\\w-]|[^\0-\\x7f])+",W="\\["+M+"*("+I+")(?:"+M+"*([*^$|!~]?=)"+M+"*(?:'((?:\\\\.|[^\\\\'])*)'|\"((?:\\\\.|[^\\\\\"])*)\"|("+I+"))|)"+M+"*\\]",F=":("+I+")(?:\\((('((?:\\\\.|[^\\\\'])*)'|\"((?:\\\\.|[^\\\\\"])*)\")|((?:\\\\.|[^\\\\()[\\]]|"+W+")*)|.*)\\)|)",B=new RegExp(M+"+","g"),$=new RegExp("^"+M+"+|((?:^|[^\\\\])(?:\\\\.)*)"+M+"+$","g"),_=new RegExp("^"+M+"*,"+M+"*"),z=new RegExp("^"+M+"*([>+~]|"+M+")"+M+"*"),U=new RegExp(M+"|>"),X=new RegExp(F),V=new RegExp("^"+I+"$"),G={ID:new RegExp("^#("+I+")"),CLASS:new RegExp("^\\.("+I+")"),TAG:new RegExp("^("+I+"|[*])"),ATTR:new RegExp("^"+W),PSEUDO:new RegExp("^"+F),CHILD:new RegExp("^:(only|first|last|nth|nth-last)-(child|of-type)(?:\\("+M+"*(even|odd|(([+-]|)(\\d*)n|)"+M+"*(?:([+-]|)"+M+"*(\\d+)|))"+M+"*\\)|)","i"),bool:new RegExp("^(?:"+R+")$","i"),needsContext:new RegExp("^"+M+"*[>+~]|:(even|odd|eq|gt|lt|nth|first|last)(?:\\("+M+"*((?:-\\d)?\\d*)"+M+"*\\)|)(?=[^-]|$)","i")},Y=/HTML$/i,Q=/^(?:input|select|textarea|button)$/i,J=/^h\d$/i,K=/^[^{]+\{\s*\[native \w/,Z=/^(?:#([\w-]+)|(\w+)|\.([\w-]+))$/,ee=/[+~]/,te=new RegExp("\\\\[\\da-fA-F]{1,6}"+M+"?|\\\\([^\\r\\n\\f])","g"),ne=function(e,t){var n="0x"+e.slice(1)-65536;return t||(n<0?String.fromCharCode(n+65536):String.fromCharCode(n>>10|55296,1023&n|56320))},re=/([\0-\x1f\x7f]|^-?\d)|^-$|[^\0-\x1f\x7f-\uFFFF\w-]/g,ie=function(e,t){return t?"\0"===e?"\ufffd":e.slice(0,-1)+"\\"+e.charCodeAt(e.length-1).toString(16)+" ":"\\"+e},oe=function(){T()},ae=be(function(e){return!0===e.disabled&&"fieldset"===e.nodeName.toLowerCase()},{dir:"parentNode",next:"legend"});try{H.apply(t=O.call(p.childNodes),p.childNodes),t[p.childNodes.length].nodeType}catch(e){H={apply:t.length?function(e,t){L.apply(e,O.call(t))}:function(e,t){var n=e.length,r=0;while(e[n++]=t[r++]);e.length=n-1}}}function se(t,e,n,r){var i,o,a,s,u,l,c,f=e&&e.ownerDocument,p=e?e.nodeType:9;if(n=n||[],"string"!=typeof t||!t||1!==p&&9!==p&&11!==p)return n;if(!r&&(T(e),e=e||C,E)){if(11!==p&&(u=Z.exec(t)))if(i=u[1]){if(9===p){if(!(a=e.getElementById(i)))return n;if(a.id===i)return n.push(a),n}else if(f&&(a=f.getElementById(i))&&y(e,a)&&a.id===i)return n.push(a),n}else{if(u[2])return H.apply(n,e.getElementsByTagName(t)),n;if((i=u[3])&&d.getElementsByClassName&&e.getElementsByClassName)return H.apply(n,e.getElementsByClassName(i)),n}if(d.qsa&&!N[t+" "]&&(!v||!v.test(t))&&(1!==p||"object"!==e.nodeName.toLowerCase())){if(c=t,f=e,1===p&&(U.test(t)||z.test(t))){(f=ee.test(t)&&ye(e.parentNode)||e)===e&&d.scope||((s=e.getAttribute("id"))?s=s.replace(re,ie):e.setAttribute("id",s=S)),o=(l=h(t)).length;while(o--)l[o]=(s?"#"+s:":scope")+" "+xe(l[o]);c=l.join(",")}try{return H.apply(n,f.querySelectorAll(c)),n}catch(e){N(t,!0)}finally{s===S&&e.removeAttribute("id")}}}return g(t.replace($,"$1"),e,n,r)}function ue(){var r=[];return function e(t,n){return r.push(t+" ")>b.cacheLength&&delete e[r.shift()],e[t+" "]=n}}function le(e){return e[S]=!0,e}function ce(e){var t=C.createElement("fieldset");try{return!!e(t)}catch(e){return!1}finally{t.parentNode&&t.parentNode.removeChild(t),t=null}}function fe(e,t){var n=e.split("|"),r=n.length;while(r--)b.attrHandle[n[r]]=t}function pe(e,t){var n=t&&e,r=n&&1===e.nodeType&&1===t.nodeType&&e.sourceIndex-t.sourceIndex;if(r)return r;if(n)while(n=n.nextSibling)if(n===t)return-1;return e?1:-1}function de(t){return function(e){return"input"===e.nodeName.toLowerCase()&&e.type===t}}function he(n){return function(e){var t=e.nodeName.toLowerCase();return("input"===t||"button"===t)&&e.type===n}}function ge(t){return function(e){return"form"in e?e.parentNode&&!1===e.disabled?"label"in e?"label"in e.parentNode?e.parentNode.disabled===t:e.disabled===t:e.isDisabled===t||e.isDisabled!==!t&&ae(e)===t:e.disabled===t:"label"in e&&e.disabled===t}}function ve(a){return le(function(o){return o=+o,le(function(e,t){var n,r=a([],e.length,o),i=r.length;while(i--)e[n=r[i]]&&(e[n]=!(t[n]=e[n]))})})}function ye(e){return e&&"undefined"!=typeof e.getElementsByTagName&&e}for(e in d=se.support={},i=se.isXML=function(e){var t=e&&e.namespaceURI,n=e&&(e.ownerDocument||e).documentElement;return!Y.test(t||n&&n.nodeName||"HTML")},T=se.setDocument=function(e){var t,n,r=e?e.ownerDocument||e:p;return r!=C&&9===r.nodeType&&r.documentElement&&(a=(C=r).documentElement,E=!i(C),p!=C&&(n=C.defaultView)&&n.top!==n&&(n.addEventListener?n.addEventListener("unload",oe,!1):n.attachEvent&&n.attachEvent("onunload",oe)),d.scope=ce(function(e){return a.appendChild(e).appendChild(C.createElement("div")),"undefined"!=typeof e.querySelectorAll&&!e.querySelectorAll(":scope fieldset div").length}),d.attributes=ce(function(e){return e.className="i",!e.getAttribute("className")}),d.getElementsByTagName=ce(function(e){return e.appendChild(C.createComment("")),!e.getElementsByTagName("*").length}),d.getElementsByClassName=K.test(C.getElementsByClassName),d.getById=ce(function(e){return a.appendChild(e).id=S,!C.getElementsByName||!C.getElementsByName(S).length}),d.getById?(b.filter.ID=function(e){var t=e.replace(te,ne);return function(e){return e.getAttribute("id")===t}},b.find.ID=function(e,t){if("undefined"!=typeof t.getElementById&&E){var n=t.getElementById(e);return n?[n]:[]}}):(b.filter.ID=function(e){var n=e.replace(te,ne);return function(e){var t="undefined"!=typeof e.getAttributeNode&&e.getAttributeNode("id");return t&&t.value===n}},b.find.ID=function(e,t){if("undefined"!=typeof t.getElementById&&E){var n,r,i,o=t.getElementById(e);if(o){if((n=o.getAttributeNode("id"))&&n.value===e)return[o];i=t.getElementsByName(e),r=0;while(o=i[r++])if((n=o.getAttributeNode("id"))&&n.value===e)return[o]}return[]}}),b.find.TAG=d.getElementsByTagName?function(e,t){return"undefined"!=typeof t.getElementsByTagName?t.getElementsByTagName(e):d.qsa?t.querySelectorAll(e):void 0}:function(e,t){var n,r=[],i=0,o=t.getElementsByTagName(e);if("*"===e){while(n=o[i++])1===n.nodeType&&r.push(n);return r}return o},b.find.CLASS=d.getElementsByClassName&&function(e,t){if("undefined"!=typeof t.getElementsByClassName&&E)return t.getElementsByClassName(e)},s=[],v=[],(d.qsa=K.test(C.querySelectorAll))&&(ce(function(e){var t;a.appendChild(e).innerHTML="<a id='"+S+"'></a><select id='"+S+"-\r\\' msallowcapture=''><option selected=''></option></select>",e.querySelectorAll("[msallowcapture^='']").length&&v.push("[*^$]="+M+"*(?:''|\"\")"),e.querySelectorAll("[selected]").length||v.push("\\["+M+"*(?:value|"+R+")"),e.querySelectorAll("[id~="+S+"-]").length||v.push("~="),(t=C.createElement("input")).setAttribute("name",""),e.appendChild(t),e.querySelectorAll("[name='']").length||v.push("\\["+M+"*name"+M+"*="+M+"*(?:''|\"\")"),e.querySelectorAll(":checked").length||v.push(":checked"),e.querySelectorAll("a#"+S+"+*").length||v.push(".#.+[+~]"),e.querySelectorAll("\\\f"),v.push("[\\r\\n\\f]")}),ce(function(e){e.innerHTML="<a href='' disabled='disabled'></a><select disabled='disabled'><option/></select>";var t=C.createElement("input");t.setAttribute("type","hidden"),e.appendChild(t).setAttribute("name","D"),e.querySelectorAll("[name=d]").length&&v.push("name"+M+"*[*^$|!~]?="),2!==e.querySelectorAll(":enabled").length&&v.push(":enabled",":disabled"),a.appendChild(e).disabled=!0,2!==e.querySelectorAll(":disabled").length&&v.push(":enabled",":disabled"),e.querySelectorAll("*,:x"),v.push(",.*:")})),(d.matchesSelector=K.test(c=a.matches||a.webkitMatchesSelector||a.mozMatchesSelector||a.oMatchesSelector||a.msMatchesSelector))&&ce(function(e){d.disconnectedMatch=c.call(e,"*"),c.call(e,"[s!='']:x"),s.push("!=",F)}),v=v.length&&new RegExp(v.join("|")),s=s.length&&new RegExp(s.join("|")),t=K.test(a.compareDocumentPosition),y=t||K.test(a.contains)?function(e,t){var n=9===e.nodeType?e.documentElement:e,r=t&&t.parentNode;return e===r||!(!r||1!==r.nodeType||!(n.contains?n.contains(r):e.compareDocumentPosition&&16&e.compareDocumentPosition(r)))}:function(e,t){if(t)while(t=t.parentNode)if(t===e)return!0;return!1},j=t?function(e,t){if(e===t)return l=!0,0;var n=!e.compareDocumentPosition-!t.compareDocumentPosition;return n||(1&(n=(e.ownerDocument||e)==(t.ownerDocument||t)?e.compareDocumentPosition(t):1)||!d.sortDetached&&t.compareDocumentPosition(e)===n?e==C||e.ownerDocument==p&&y(p,e)?-1:t==C||t.ownerDocument==p&&y(p,t)?1:u?P(u,e)-P(u,t):0:4&n?-1:1)}:function(e,t){if(e===t)return l=!0,0;var n,r=0,i=e.parentNode,o=t.parentNode,a=[e],s=[t];if(!i||!o)return e==C?-1:t==C?1:i?-1:o?1:u?P(u,e)-P(u,t):0;if(i===o)return pe(e,t);n=e;while(n=n.parentNode)a.unshift(n);n=t;while(n=n.parentNode)s.unshift(n);while(a[r]===s[r])r++;return r?pe(a[r],s[r]):a[r]==p?-1:s[r]==p?1:0}),C},se.matches=function(e,t){return se(e,null,null,t)},se.matchesSelector=function(e,t){if(T(e),d.matchesSelector&&E&&!N[t+" "]&&(!s||!s.test(t))&&(!v||!v.test(t)))try{var n=c.call(e,t);if(n||d.disconnectedMatch||e.document&&11!==e.document.nodeType)return n}catch(e){N(t,!0)}return 0<se(t,C,null,[e]).length},se.contains=function(e,t){return(e.ownerDocument||e)!=C&&T(e),y(e,t)},se.attr=function(e,t){(e.ownerDocument||e)!=C&&T(e);var n=b.attrHandle[t.toLowerCase()],r=n&&D.call(b.attrHandle,t.toLowerCase())?n(e,t,!E):void 0;return void 0!==r?r:d.attributes||!E?e.getAttribute(t):(r=e.getAttributeNode(t))&&r.specified?r.value:null},se.escape=function(e){return(e+"").replace(re,ie)},se.error=function(e){throw new Error("Syntax error, unrecognized expression: "+e)},se.uniqueSort=function(e){var t,n=[],r=0,i=0;if(l=!d.detectDuplicates,u=!d.sortStable&&e.slice(0),e.sort(j),l){while(t=e[i++])t===e[i]&&(r=n.push(i));while(r--)e.splice(n[r],1)}return u=null,e},o=se.getText=function(e){var t,n="",r=0,i=e.nodeType;if(i){if(1===i||9===i||11===i){if("string"==typeof e.textContent)return e.textContent;for(e=e.firstChild;e;e=e.nextSibling)n+=o(e)}else if(3===i||4===i)return e.nodeValue}else while(t=e[r++])n+=o(t);return n},(b=se.selectors={cacheLength:50,createPseudo:le,match:G,attrHandle:{},find:{},relative:{">":{dir:"parentNode",first:!0}," ":{dir:"parentNode"},"+":{dir:"previousSibling",first:!0},"~":{dir:"previousSibling"}},preFilter:{ATTR:function(e){return e[1]=e[1].replace(te,ne),e[3]=(e[3]||e[4]||e[5]||"").replace(te,ne),"~="===e[2]&&(e[3]=" "+e[3]+" "),e.slice(0,4)},CHILD:function(e){return e[1]=e[1].toLowerCase(),"nth"===e[1].slice(0,3)?(e[3]||se.error(e[0]),e[4]=+(e[4]?e[5]+(e[6]||1):2*("even"===e[3]||"odd"===e[3])),e[5]=+(e[7]+e[8]||"odd"===e[3])):e[3]&&se.error(e[0]),e},PSEUDO:function(e){var t,n=!e[6]&&e[2];return G.CHILD.test(e[0])?null:(e[3]?e[2]=e[4]||e[5]||"":n&&X.test(n)&&(t=h(n,!0))&&(t=n.indexOf(")",n.length-t)-n.length)&&(e[0]=e[0].slice(0,t),e[2]=n.slice(0,t)),e.slice(0,3))}},filter:{TAG:function(e){var t=e.replace(te,ne).toLowerCase();return"*"===e?function(){return!0}:function(e){return e.nodeName&&e.nodeName.toLowerCase()===t}},CLASS:function(e){var t=m[e+" "];return t||(t=new RegExp("(^|"+M+")"+e+"("+M+"|$)"))&&m(e,function(e){return t.test("string"==typeof e.className&&e.className||"undefined"!=typeof e.getAttribute&&e.getAttribute("class")||"")})},ATTR:function(n,r,i){return function(e){var t=se.attr(e,n);return null==t?"!="===r:!r||(t+="","="===r?t===i:"!="===r?t!==i:"^="===r?i&&0===t.indexOf(i):"*="===r?i&&-1<t.indexOf(i):"$="===r?i&&t.slice(-i.length)===i:"~="===r?-1<(" "+t.replace(B," ")+" ").indexOf(i):"|="===r&&(t===i||t.slice(0,i.length+1)===i+"-"))}},CHILD:function(h,e,t,g,v){var y="nth"!==h.slice(0,3),m="last"!==h.slice(-4),x="of-type"===e;return 1===g&&0===v?function(e){return!!e.parentNode}:function(e,t,n){var r,i,o,a,s,u,l=y!==m?"nextSibling":"previousSibling",c=e.parentNode,f=x&&e.nodeName.toLowerCase(),p=!n&&!x,d=!1;if(c){if(y){while(l){a=e;while(a=a[l])if(x?a.nodeName.toLowerCase()===f:1===a.nodeType)return!1;u=l="only"===h&&!u&&"nextSibling"}return!0}if(u=[m?c.firstChild:c.lastChild],m&&p){d=(s=(r=(i=(o=(a=c)[S]||(a[S]={}))[a.uniqueID]||(o[a.uniqueID]={}))[h]||[])[0]===k&&r[1])&&r[2],a=s&&c.childNodes[s];while(a=++s&&a&&a[l]||(d=s=0)||u.pop())if(1===a.nodeType&&++d&&a===e){i[h]=[k,s,d];break}}else if(p&&(d=s=(r=(i=(o=(a=e)[S]||(a[S]={}))[a.uniqueID]||(o[a.uniqueID]={}))[h]||[])[0]===k&&r[1]),!1===d)while(a=++s&&a&&a[l]||(d=s=0)||u.pop())if((x?a.nodeName.toLowerCase()===f:1===a.nodeType)&&++d&&(p&&((i=(o=a[S]||(a[S]={}))[a.uniqueID]||(o[a.uniqueID]={}))[h]=[k,d]),a===e))break;return(d-=v)===g||d%g==0&&0<=d/g}}},PSEUDO:function(e,o){var t,a=b.pseudos[e]||b.setFilters[e.toLowerCase()]||se.error("unsupported pseudo: "+e);return a[S]?a(o):1<a.length?(t=[e,e,"",o],b.setFilters.hasOwnProperty(e.toLowerCase())?le(function(e,t){var n,r=a(e,o),i=r.length;while(i--)e[n=P(e,r[i])]=!(t[n]=r[i])}):function(e){return a(e,0,t)}):a}},pseudos:{not:le(function(e){var r=[],i=[],s=f(e.replace($,"$1"));return s[S]?le(function(e,t,n,r){var i,o=s(e,null,r,[]),a=e.length;while(a--)(i=o[a])&&(e[a]=!(t[a]=i))}):function(e,t,n){return r[0]=e,s(r,null,n,i),r[0]=null,!i.pop()}}),has:le(function(t){return function(e){return 0<se(t,e).length}}),contains:le(function(t){return t=t.replace(te,ne),function(e){return-1<(e.textContent||o(e)).indexOf(t)}}),lang:le(function(n){return V.test(n||"")||se.error("unsupported lang: "+n),n=n.replace(te,ne).toLowerCase(),function(e){var t;do{if(t=E?e.lang:e.getAttribute("xml:lang")||e.getAttribute("lang"))return(t=t.toLowerCase())===n||0===t.indexOf(n+"-")}while((e=e.parentNode)&&1===e.nodeType);return!1}}),target:function(e){var t=n.location&&n.location.hash;return t&&t.slice(1)===e.id},root:function(e){return e===a},focus:function(e){return e===C.activeElement&&(!C.hasFocus||C.hasFocus())&&!!(e.type||e.href||~e.tabIndex)},enabled:ge(!1),disabled:ge(!0),checked:function(e){var t=e.nodeName.toLowerCase();return"input"===t&&!!e.checked||"option"===t&&!!e.selected},selected:function(e){return e.parentNode&&e.parentNode.selectedIndex,!0===e.selected},empty:function(e){for(e=e.firstChild;e;e=e.nextSibling)if(e.nodeType<6)return!1;return!0},parent:function(e){return!b.pseudos.empty(e)},header:function(e){return J.test(e.nodeName)},input:function(e){return Q.test(e.nodeName)},button:function(e){var t=e.nodeName.toLowerCase();return"input"===t&&"button"===e.type||"button"===t},text:function(e){var t;return"input"===e.nodeName.toLowerCase()&&"text"===e.type&&(null==(t=e.getAttribute("type"))||"text"===t.toLowerCase())},first:ve(function(){return[0]}),last:ve(function(e,t){return[t-1]}),eq:ve(function(e,t,n){return[n<0?n+t:n]}),even:ve(function(e,t){for(var n=0;n<t;n+=2)e.push(n);return e}),odd:ve(function(e,t){for(var n=1;n<t;n+=2)e.push(n);return e}),lt:ve(function(e,t,n){for(var r=n<0?n+t:t<n?t:n;0<=--r;)e.push(r);return e}),gt:ve(function(e,t,n){for(var r=n<0?n+t:n;++r<t;)e.push(r);return e})}}).pseudos.nth=b.pseudos.eq,{radio:!0,checkbox:!0,file:!0,password:!0,image:!0})b.pseudos[e]=de(e);for(e in{submit:!0,reset:!0})b.pseudos[e]=he(e);function me(){}function xe(e){for(var t=0,n=e.length,r="";t<n;t++)r+=e[t].value;return r}function be(s,e,t){var u=e.dir,l=e.next,c=l||u,f=t&&"parentNode"===c,p=r++;return e.first?function(e,t,n){while(e=e[u])if(1===e.nodeType||f)return s(e,t,n);return!1}:function(e,t,n){var r,i,o,a=[k,p];if(n){while(e=e[u])if((1===e.nodeType||f)&&s(e,t,n))return!0}else while(e=e[u])if(1===e.nodeType||f)if(i=(o=e[S]||(e[S]={}))[e.uniqueID]||(o[e.uniqueID]={}),l&&l===e.nodeName.toLowerCase())e=e[u]||e;else{if((r=i[c])&&r[0]===k&&r[1]===p)return a[2]=r[2];if((i[c]=a)[2]=s(e,t,n))return!0}return!1}}function we(i){return 1<i.length?function(e,t,n){var r=i.length;while(r--)if(!i[r](e,t,n))return!1;return!0}:i[0]}function Te(e,t,n,r,i){for(var o,a=[],s=0,u=e.length,l=null!=t;s<u;s++)(o=e[s])&&(n&&!n(o,r,i)||(a.push(o),l&&t.push(s)));return a}function Ce(d,h,g,v,y,e){return v&&!v[S]&&(v=Ce(v)),y&&!y[S]&&(y=Ce(y,e)),le(function(e,t,n,r){var i,o,a,s=[],u=[],l=t.length,c=e||function(e,t,n){for(var r=0,i=t.length;r<i;r++)se(e,t[r],n);return n}(h||"*",n.nodeType?[n]:n,[]),f=!d||!e&&h?c:Te(c,s,d,n,r),p=g?y||(e?d:l||v)?[]:t:f;if(g&&g(f,p,n,r),v){i=Te(p,u),v(i,[],n,r),o=i.length;while(o--)(a=i[o])&&(p[u[o]]=!(f[u[o]]=a))}if(e){if(y||d){if(y){i=[],o=p.length;while(o--)(a=p[o])&&i.push(f[o]=a);y(null,p=[],i,r)}o=p.length;while(o--)(a=p[o])&&-1<(i=y?P(e,a):s[o])&&(e[i]=!(t[i]=a))}}else p=Te(p===t?p.splice(l,p.length):p),y?y(null,t,p,r):H.apply(t,p)})}function Ee(e){for(var i,t,n,r=e.length,o=b.relative[e[0].type],a=o||b.relative[" "],s=o?1:0,u=be(function(e){return e===i},a,!0),l=be(function(e){return-1<P(i,e)},a,!0),c=[function(e,t,n){var r=!o&&(n||t!==w)||((i=t).nodeType?u(e,t,n):l(e,t,n));return i=null,r}];s<r;s++)if(t=b.relative[e[s].type])c=[be(we(c),t)];else{if((t=b.filter[e[s].type].apply(null,e[s].matches))[S]){for(n=++s;n<r;n++)if(b.relative[e[n].type])break;return Ce(1<s&&we(c),1<s&&xe(e.slice(0,s-1).concat({value:" "===e[s-2].type?"*":""})).replace($,"$1"),t,s<n&&Ee(e.slice(s,n)),n<r&&Ee(e=e.slice(n)),n<r&&xe(e))}c.push(t)}return we(c)}return me.prototype=b.filters=b.pseudos,b.setFilters=new me,h=se.tokenize=function(e,t){var n,r,i,o,a,s,u,l=x[e+" "];if(l)return t?0:l.slice(0);a=e,s=[],u=b.preFilter;while(a){for(o in n&&!(r=_.exec(a))||(r&&(a=a.slice(r[0].length)||a),s.push(i=[])),n=!1,(r=z.exec(a))&&(n=r.shift(),i.push({value:n,type:r[0].replace($," ")}),a=a.slice(n.length)),b.filter)!(r=G[o].exec(a))||u[o]&&!(r=u[o](r))||(n=r.shift(),i.push({value:n,type:o,matches:r}),a=a.slice(n.length));if(!n)break}return t?a.length:a?se.error(e):x(e,s).slice(0)},f=se.compile=function(e,t){var n,v,y,m,x,r,i=[],o=[],a=A[e+" "];if(!a){t||(t=h(e)),n=t.length;while(n--)(a=Ee(t[n]))[S]?i.push(a):o.push(a);(a=A(e,(v=o,m=0<(y=i).length,x=0<v.length,r=function(e,t,n,r,i){var o,a,s,u=0,l="0",c=e&&[],f=[],p=w,d=e||x&&b.find.TAG("*",i),h=k+=null==p?1:Math.random()||.1,g=d.length;for(i&&(w=t==C||t||i);l!==g&&null!=(o=d[l]);l++){if(x&&o){a=0,t||o.ownerDocument==C||(T(o),n=!E);while(s=v[a++])if(s(o,t||C,n)){r.push(o);break}i&&(k=h)}m&&((o=!s&&o)&&u--,e&&c.push(o))}if(u+=l,m&&l!==u){a=0;while(s=y[a++])s(c,f,t,n);if(e){if(0<u)while(l--)c[l]||f[l]||(f[l]=q.call(r));f=Te(f)}H.apply(r,f),i&&!e&&0<f.length&&1<u+y.length&&se.uniqueSort(r)}return i&&(k=h,w=p),c},m?le(r):r))).selector=e}return a},g=se.select=function(e,t,n,r){var i,o,a,s,u,l="function"==typeof e&&e,c=!r&&h(e=l.selector||e);if(n=n||[],1===c.length){if(2<(o=c[0]=c[0].slice(0)).length&&"ID"===(a=o[0]).type&&9===t.nodeType&&E&&b.relative[o[1].type]){if(!(t=(b.find.ID(a.matches[0].replace(te,ne),t)||[])[0]))return n;l&&(t=t.parentNode),e=e.slice(o.shift().value.length)}i=G.needsContext.test(e)?0:o.length;while(i--){if(a=o[i],b.relative[s=a.type])break;if((u=b.find[s])&&(r=u(a.matches[0].replace(te,ne),ee.test(o[0].type)&&ye(t.parentNode)||t))){if(o.splice(i,1),!(e=r.length&&xe(o)))return H.apply(n,r),n;break}}}return(l||f(e,c))(r,t,!E,n,!t||ee.test(e)&&ye(t.parentNode)||t),n},d.sortStable=S.split("").sort(j).join("")===S,d.detectDuplicates=!!l,T(),d.sortDetached=ce(function(e){return 1&e.compareDocumentPosition(C.createElement("fieldset"))}),ce(function(e){return e.innerHTML="<a href='#'></a>","#"===e.firstChild.getAttribute("href")})||fe("type|href|height|width",function(e,t,n){if(!n)return e.getAttribute(t,"type"===t.toLowerCase()?1:2)}),d.attributes&&ce(function(e){return e.innerHTML="<input/>",e.firstChild.setAttribute("value",""),""===e.firstChild.getAttribute("value")})||fe("value",function(e,t,n){if(!n&&"input"===e.nodeName.toLowerCase())return e.defaultValue}),ce(function(e){return null==e.getAttribute("disabled")})||fe(R,function(e,t,n){var r;if(!n)return!0===e[t]?t.toLowerCase():(r=e.getAttributeNode(t))&&r.specified?r.value:null}),se}(C);S.find=d,S.expr=d.selectors,S.expr[":"]=S.expr.pseudos,S.uniqueSort=S.unique=d.uniqueSort,S.text=d.getText,S.isXMLDoc=d.isXML,S.contains=d.contains,S.escapeSelector=d.escape;var h=function(e,t,n){var r=[],i=void 0!==n;while((e=e[t])&&9!==e.nodeType)if(1===e.nodeType){if(i&&S(e).is(n))break;r.push(e)}return r},T=function(e,t){for(var n=[];e;e=e.nextSibling)1===e.nodeType&&e!==t&&n.push(e);return n},k=S.expr.match.needsContext;function A(e,t){return e.nodeName&&e.nodeName.toLowerCase()===t.toLowerCase()}var N=/^<([a-z][^\/\0>:\x20\t\r\n\f]*)[\x20\t\r\n\f]*\/?>(?:<\/\1>|)$/i;function j(e,n,r){return m(n)?S.grep(e,function(e,t){return!!n.call(e,t,e)!==r}):n.nodeType?S.grep(e,function(e){return e===n!==r}):"string"!=typeof n?S.grep(e,function(e){return-1<i.call(n,e)!==r}):S.filter(n,e,r)}S.filter=function(e,t,n){var r=t[0];return n&&(e=":not("+e+")"),1===t.length&&1===r.nodeType?S.find.matchesSelector(r,e)?[r]:[]:S.find.matches(e,S.grep(t,function(e){return 1===e.nodeType}))},S.fn.extend({find:function(e){var t,n,r=this.length,i=this;if("string"!=typeof e)return this.pushStack(S(e).filter(function(){for(t=0;t<r;t++)if(S.contains(i[t],this))return!0}));for(n=this.pushStack([]),t=0;t<r;t++)S.find(e,i[t],n);return 1<r?S.uniqueSort(n):n},filter:function(e){return this.pushStack(j(this,e||[],!1))},not:function(e){return this.pushStack(j(this,e||[],!0))},is:function(e){return!!j(this,"string"==typeof e&&k.test(e)?S(e):e||[],!1).length}});var D,q=/^(?:\s*(<[\w\W]+>)[^>]*|#([\w-]+))$/;(S.fn.init=function(e,t,n){var r,i;if(!e)return this;if(n=n||D,"string"==typeof e){if(!(r="<"===e[0]&&">"===e[e.length-1]&&3<=e.length?[null,e,null]:q.exec(e))||!r[1]&&t)return!t||t.jquery?(t||n).find(e):this.constructor(t).find(e);if(r[1]){if(t=t instanceof S?t[0]:t,S.merge(this,S.parseHTML(r[1],t&&t.nodeType?t.ownerDocument||t:E,!0)),N.test(r[1])&&S.isPlainObject(t))for(r in t)m(this[r])?this[r](t[r]):this.attr(r,t[r]);return this}return(i=E.getElementById(r[2]))&&(this[0]=i,this.length=1),this}return e.nodeType?(this[0]=e,this.length=1,this):m(e)?void 0!==n.ready?n.ready(e):e(S):S.makeArray(e,this)}).prototype=S.fn,D=S(E);var L=/^(?:parents|prev(?:Until|All))/,H={children:!0,contents:!0,next:!0,prev:!0};function O(e,t){while((e=e[t])&&1!==e.nodeType);return e}S.fn.extend({has:function(e){var t=S(e,this),n=t.length;return this.filter(function(){for(var e=0;e<n;e++)if(S.contains(this,t[e]))return!0})},closest:function(e,t){var n,r=0,i=this.length,o=[],a="string"!=typeof e&&S(e);if(!k.test(e))for(;r<i;r++)for(n=this[r];n&&n!==t;n=n.parentNode)if(n.nodeType<11&&(a?-1<a.index(n):1===n.nodeType&&S.find.matchesSelector(n,e))){o.push(n);break}return this.pushStack(1<o.length?S.uniqueSort(o):o)},index:function(e){return e?"string"==typeof e?i.call(S(e),this[0]):i.call(this,e.jquery?e[0]:e):this[0]&&this[0].parentNode?this.first().prevAll().length:-1},add:function(e,t){return this.pushStack(S.uniqueSort(S.merge(this.get(),S(e,t))))},addBack:function(e){return this.add(null==e?this.prevObject:this.prevObject.filter(e))}}),S.each({parent:function(e){var t=e.parentNode;return t&&11!==t.nodeType?t:null},parents:function(e){return h(e,"parentNode")},parentsUntil:function(e,t,n){return h(e,"parentNode",n)},next:function(e){return O(e,"nextSibling")},prev:function(e){return O(e,"previousSibling")},nextAll:function(e){return h(e,"nextSibling")},prevAll:function(e){return h(e,"previousSibling")},nextUntil:function(e,t,n){return h(e,"nextSibling",n)},prevUntil:function(e,t,n){return h(e,"previousSibling",n)},siblings:function(e){return T((e.parentNode||{}).firstChild,e)},children:function(e){return T(e.firstChild)},contents:function(e){return null!=e.contentDocument&&r(e.contentDocument)?e.contentDocument:(A(e,"template")&&(e=e.content||e),S.merge([],e.childNodes))}},function(r,i){S.fn[r]=function(e,t){var n=S.map(this,i,e);return"Until"!==r.slice(-5)&&(t=e),t&&"string"==typeof t&&(n=S.filter(t,n)),1<this.length&&(H[r]||S.uniqueSort(n),L.test(r)&&n.reverse()),this.pushStack(n)}});var P=/[^\x20\t\r\n\f]+/g;function R(e){return e}function M(e){throw e}function I(e,t,n,r){var i;try{e&&m(i=e.promise)?i.call(e).done(t).fail(n):e&&m(i=e.then)?i.call(e,t,n):t.apply(void 0,[e].slice(r))}catch(e){n.apply(void 0,[e])}}S.Callbacks=function(r){var e,n;r="string"==typeof r?(e=r,n={},S.each(e.match(P)||[],function(e,t){n[t]=!0}),n):S.extend({},r);var i,t,o,a,s=[],u=[],l=-1,c=function(){for(a=a||r.once,o=i=!0;u.length;l=-1){t=u.shift();while(++l<s.length)!1===s[l].apply(t[0],t[1])&&r.stopOnFalse&&(l=s.length,t=!1)}r.memory||(t=!1),i=!1,a&&(s=t?[]:"")},f={add:function(){return s&&(t&&!i&&(l=s.length-1,u.push(t)),function n(e){S.each(e,function(e,t){m(t)?r.unique&&f.has(t)||s.push(t):t&&t.length&&"string"!==w(t)&&n(t)})}(arguments),t&&!i&&c()),this},remove:function(){return S.each(arguments,function(e,t){var n;while(-1<(n=S.inArray(t,s,n)))s.splice(n,1),n<=l&&l--}),this},has:function(e){return e?-1<S.inArray(e,s):0<s.length},empty:function(){return s&&(s=[]),this},disable:function(){return a=u=[],s=t="",this},disabled:function(){return!s},lock:function(){return a=u=[],t||i||(s=t=""),this},locked:function(){return!!a},fireWith:function(e,t){return a||(t=[e,(t=t||[]).slice?t.slice():t],u.push(t),i||c()),this},fire:function(){return f.fireWith(this,arguments),this},fired:function(){return!!o}};return f},S.extend({Deferred:function(e){var o=[["notify","progress",S.Callbacks("memory"),S.Callbacks("memory"),2],["resolve","done",S.Callbacks("once memory"),S.Callbacks("once memory"),0,"resolved"],["reject","fail",S.Callbacks("once memory"),S.Callbacks("once memory"),1,"rejected"]],i="pending",a={state:function(){return i},always:function(){return s.done(arguments).fail(arguments),this},"catch":function(e){return a.then(null,e)},pipe:function(){var i=arguments;return S.Deferred(function(r){S.each(o,function(e,t){var n=m(i[t[4]])&&i[t[4]];s[t[1]](function(){var e=n&&n.apply(this,arguments);e&&m(e.promise)?e.promise().progress(r.notify).done(r.resolve).fail(r.reject):r[t[0]+"With"](this,n?[e]:arguments)})}),i=null}).promise()},then:function(t,n,r){var u=0;function l(i,o,a,s){return function(){var n=this,r=arguments,e=function(){var e,t;if(!(i<u)){if((e=a.apply(n,r))===o.promise())throw new TypeError("Thenable self-resolution");t=e&&("object"==typeof e||"function"==typeof e)&&e.then,m(t)?s?t.call(e,l(u,o,R,s),l(u,o,M,s)):(u++,t.call(e,l(u,o,R,s),l(u,o,M,s),l(u,o,R,o.notifyWith))):(a!==R&&(n=void 0,r=[e]),(s||o.resolveWith)(n,r))}},t=s?e:function(){try{e()}catch(e){S.Deferred.exceptionHook&&S.Deferred.exceptionHook(e,t.stackTrace),u<=i+1&&(a!==M&&(n=void 0,r=[e]),o.rejectWith(n,r))}};i?t():(S.Deferred.getStackHook&&(t.stackTrace=S.Deferred.getStackHook()),C.setTimeout(t))}}return S.Deferred(function(e){o[0][3].add(l(0,e,m(r)?r:R,e.notifyWith)),o[1][3].add(l(0,e,m(t)?t:R)),o[2][3].add(l(0,e,m(n)?n:M))}).promise()},promise:function(e){return null!=e?S.extend(e,a):a}},s={};return S.each(o,function(e,t){var n=t[2],r=t[5];a[t[1]]=n.add,r&&n.add(function(){i=r},o[3-e][2].disable,o[3-e][3].disable,o[0][2].lock,o[0][3].lock),n.add(t[3].fire),s[t[0]]=function(){return s[t[0]+"With"](this===s?void 0:this,arguments),this},s[t[0]+"With"]=n.fireWith}),a.promise(s),e&&e.call(s,s),s},when:function(e){var n=arguments.length,t=n,r=Array(t),i=s.call(arguments),o=S.Deferred(),a=function(t){return function(e){r[t]=this,i[t]=1<arguments.length?s.call(arguments):e,--n||o.resolveWith(r,i)}};if(n<=1&&(I(e,o.done(a(t)).resolve,o.reject,!n),"pending"===o.state()||m(i[t]&&i[t].then)))return o.then();while(t--)I(i[t],a(t),o.reject);return o.promise()}});var W=/^(Eval|Internal|Range|Reference|Syntax|Type|URI)Error$/;S.Deferred.exceptionHook=function(e,t){C.console&&C.console.warn&&e&&W.test(e.name)&&C.console.warn("jQuery.Deferred exception: "+e.message,e.stack,t)},S.readyException=function(e){C.setTimeout(function(){throw e})};var F=S.Deferred();function B(){E.removeEventListener("DOMContentLoaded",B),C.removeEventListener("load",B),S.ready()}S.fn.ready=function(e){return F.then(e)["catch"](function(e){S.readyException(e)}),this},S.extend({isReady:!1,readyWait:1,ready:function(e){(!0===e?--S.readyWait:S.isReady)||(S.isReady=!0)!==e&&0<--S.readyWait||F.resolveWith(E,[S])}}),S.ready.then=F.then,"complete"===E.readyState||"loading"!==E.readyState&&!E.documentElement.doScroll?C.setTimeout(S.ready):(E.addEventListener("DOMContentLoaded",B),C.addEventListener("load",B));var $=function(e,t,n,r,i,o,a){var s=0,u=e.length,l=null==n;if("object"===w(n))for(s in i=!0,n)$(e,t,s,n[s],!0,o,a);else if(void 0!==r&&(i=!0,m(r)||(a=!0),l&&(a?(t.call(e,r),t=null):(l=t,t=function(e,t,n){return l.call(S(e),n)})),t))for(;s<u;s++)t(e[s],n,a?r:r.call(e[s],s,t(e[s],n)));return i?e:l?t.call(e):u?t(e[0],n):o},_=/^-ms-/,z=/-([a-z])/g;function U(e,t){return t.toUpperCase()}function X(e){return e.replace(_,"ms-").replace(z,U)}var V=function(e){return 1===e.nodeType||9===e.nodeType||!+e.nodeType};function G(){this.expando=S.expando+G.uid++}G.uid=1,G.prototype={cache:function(e){var t=e[this.expando];return t||(t={},V(e)&&(e.nodeType?e[this.expando]=t:Object.defineProperty(e,this.expando,{value:t,configurable:!0}))),t},set:function(e,t,n){var r,i=this.cache(e);if("string"==typeof t)i[X(t)]=n;else for(r in t)i[X(r)]=t[r];return i},get:function(e,t){return void 0===t?this.cache(e):e[this.expando]&&e[this.expando][X(t)]},access:function(e,t,n){return void 0===t||t&&"string"==typeof t&&void 0===n?this.get(e,t):(this.set(e,t,n),void 0!==n?n:t)},remove:function(e,t){var n,r=e[this.expando];if(void 0!==r){if(void 0!==t){n=(t=Array.isArray(t)?t.map(X):(t=X(t))in r?[t]:t.match(P)||[]).length;while(n--)delete r[t[n]]}(void 0===t||S.isEmptyObject(r))&&(e.nodeType?e[this.expando]=void 0:delete e[this.expando])}},hasData:function(e){var t=e[this.expando];return void 0!==t&&!S.isEmptyObject(t)}};var Y=new G,Q=new G,J=/^(?:\{[\w\W]*\}|\[[\w\W]*\])$/,K=/[A-Z]/g;function Z(e,t,n){var r,i;if(void 0===n&&1===e.nodeType)if(r="data-"+t.replace(K,"-$&").toLowerCase(),"string"==typeof(n=e.getAttribute(r))){try{n="true"===(i=n)||"false"!==i&&("null"===i?null:i===+i+""?+i:J.test(i)?JSON.parse(i):i)}catch(e){}Q.set(e,t,n)}else n=void 0;return n}S.extend({hasData:function(e){return Q.hasData(e)||Y.hasData(e)},data:function(e,t,n){return Q.access(e,t,n)},removeData:function(e,t){Q.remove(e,t)},_data:function(e,t,n){return Y.access(e,t,n)},_removeData:function(e,t){Y.remove(e,t)}}),S.fn.extend({data:function(n,e){var t,r,i,o=this[0],a=o&&o.attributes;if(void 0===n){if(this.length&&(i=Q.get(o),1===o.nodeType&&!Y.get(o,"hasDataAttrs"))){t=a.length;while(t--)a[t]&&0===(r=a[t].name).indexOf("data-")&&(r=X(r.slice(5)),Z(o,r,i[r]));Y.set(o,"hasDataAttrs",!0)}return i}return"object"==typeof n?this.each(function(){Q.set(this,n)}):$(this,function(e){var t;if(o&&void 0===e)return void 0!==(t=Q.get(o,n))?t:void 0!==(t=Z(o,n))?t:void 0;this.each(function(){Q.set(this,n,e)})},null,e,1<arguments.length,null,!0)},removeData:function(e){return this.each(function(){Q.remove(this,e)})}}),S.extend({queue:function(e,t,n){var r;if(e)return t=(t||"fx")+"queue",r=Y.get(e,t),n&&(!r||Array.isArray(n)?r=Y.access(e,t,S.makeArray(n)):r.push(n)),r||[]},dequeue:function(e,t){t=t||"fx";var n=S.queue(e,t),r=n.length,i=n.shift(),o=S._queueHooks(e,t);"inprogress"===i&&(i=n.shift(),r--),i&&("fx"===t&&n.unshift("inprogress"),delete o.stop,i.call(e,function(){S.dequeue(e,t)},o)),!r&&o&&o.empty.fire()},_queueHooks:function(e,t){var n=t+"queueHooks";return Y.get(e,n)||Y.access(e,n,{empty:S.Callbacks("once memory").add(function(){Y.remove(e,[t+"queue",n])})})}}),S.fn.extend({queue:function(t,n){var e=2;return"string"!=typeof t&&(n=t,t="fx",e--),arguments.length<e?S.queue(this[0],t):void 0===n?this:this.each(function(){var e=S.queue(this,t,n);S._queueHooks(this,t),"fx"===t&&"inprogress"!==e[0]&&S.dequeue(this,t)})},dequeue:function(e){return this.each(function(){S.dequeue(this,e)})},clearQueue:function(e){return this.queue(e||"fx",[])},promise:function(e,t){var n,r=1,i=S.Deferred(),o=this,a=this.length,s=function(){--r||i.resolveWith(o,[o])};"string"!=typeof e&&(t=e,e=void 0),e=e||"fx";while(a--)(n=Y.get(o[a],e+"queueHooks"))&&n.empty&&(r++,n.empty.add(s));return s(),i.promise(t)}});var ee=/[+-]?(?:\d*\.|)\d+(?:[eE][+-]?\d+|)/.source,te=new RegExp("^(?:([+-])=|)("+ee+")([a-z%]*)$","i"),ne=["Top","Right","Bottom","Left"],re=E.documentElement,ie=function(e){return S.contains(e.ownerDocument,e)},oe={composed:!0};re.getRootNode&&(ie=function(e){return S.contains(e.ownerDocument,e)||e.getRootNode(oe)===e.ownerDocument});var ae=function(e,t){return"none"===(e=t||e).style.display||""===e.style.display&&ie(e)&&"none"===S.css(e,"display")};function se(e,t,n,r){var i,o,a=20,s=r?function(){return r.cur()}:function(){return S.css(e,t,"")},u=s(),l=n&&n[3]||(S.cssNumber[t]?"":"px"),c=e.nodeType&&(S.cssNumber[t]||"px"!==l&&+u)&&te.exec(S.css(e,t));if(c&&c[3]!==l){u/=2,l=l||c[3],c=+u||1;while(a--)S.style(e,t,c+l),(1-o)*(1-(o=s()/u||.5))<=0&&(a=0),c/=o;c*=2,S.style(e,t,c+l),n=n||[]}return n&&(c=+c||+u||0,i=n[1]?c+(n[1]+1)*n[2]:+n[2],r&&(r.unit=l,r.start=c,r.end=i)),i}var ue={};function le(e,t){for(var n,r,i,o,a,s,u,l=[],c=0,f=e.length;c<f;c++)(r=e[c]).style&&(n=r.style.display,t?("none"===n&&(l[c]=Y.get(r,"display")||null,l[c]||(r.style.display="")),""===r.style.display&&ae(r)&&(l[c]=(u=a=o=void 0,a=(i=r).ownerDocument,s=i.nodeName,(u=ue[s])||(o=a.body.appendChild(a.createElement(s)),u=S.css(o,"display"),o.parentNode.removeChild(o),"none"===u&&(u="block"),ue[s]=u)))):"none"!==n&&(l[c]="none",Y.set(r,"display",n)));for(c=0;c<f;c++)null!=l[c]&&(e[c].style.display=l[c]);return e}S.fn.extend({show:function(){return le(this,!0)},hide:function(){return le(this)},toggle:function(e){return"boolean"==typeof e?e?this.show():this.hide():this.each(function(){ae(this)?S(this).show():S(this).hide()})}});var ce,fe,pe=/^(?:checkbox|radio)$/i,de=/<([a-z][^\/\0>\x20\t\r\n\f]*)/i,he=/^$|^module$|\/(?:java|ecma)script/i;ce=E.createDocumentFragment().appendChild(E.createElement("div")),(fe=E.createElement("input")).setAttribute("type","radio"),fe.setAttribute("checked","checked"),fe.setAttribute("name","t"),ce.appendChild(fe),y.checkClone=ce.cloneNode(!0).cloneNode(!0).lastChild.checked,ce.innerHTML="<textarea>x</textarea>",y.noCloneChecked=!!ce.cloneNode(!0).lastChild.defaultValue,ce.innerHTML="<option></option>",y.option=!!ce.lastChild;var ge={thead:[1,"<table>","</table>"],col:[2,"<table><colgroup>","</colgroup></table>"],tr:[2,"<table><tbody>","</tbody></table>"],td:[3,"<table><tbody><tr>","</tr></tbody></table>"],_default:[0,"",""]};function ve(e,t){var n;return n="undefined"!=typeof e.getElementsByTagName?e.getElementsByTagName(t||"*"):"undefined"!=typeof e.querySelectorAll?e.querySelectorAll(t||"*"):[],void 0===t||t&&A(e,t)?S.merge([e],n):n}function ye(e,t){for(var n=0,r=e.length;n<r;n++)Y.set(e[n],"globalEval",!t||Y.get(t[n],"globalEval"))}ge.tbody=ge.tfoot=ge.colgroup=ge.caption=ge.thead,ge.th=ge.td,y.option||(ge.optgroup=ge.option=[1,"<select multiple='multiple'>","</select>"]);var me=/<|&#?\w+;/;function xe(e,t,n,r,i){for(var o,a,s,u,l,c,f=t.createDocumentFragment(),p=[],d=0,h=e.length;d<h;d++)if((o=e[d])||0===o)if("object"===w(o))S.merge(p,o.nodeType?[o]:o);else if(me.test(o)){a=a||f.appendChild(t.createElement("div")),s=(de.exec(o)||["",""])[1].toLowerCase(),u=ge[s]||ge._default,a.innerHTML=u[1]+S.htmlPrefilter(o)+u[2],c=u[0];while(c--)a=a.lastChild;S.merge(p,a.childNodes),(a=f.firstChild).textContent=""}else p.push(t.createTextNode(o));f.textContent="",d=0;while(o=p[d++])if(r&&-1<S.inArray(o,r))i&&i.push(o);else if(l=ie(o),a=ve(f.appendChild(o),"script"),l&&ye(a),n){c=0;while(o=a[c++])he.test(o.type||"")&&n.push(o)}return f}var be=/^([^.]*)(?:\.(.+)|)/;function we(){return!0}function Te(){return!1}function Ce(e,t){return e===function(){try{return E.activeElement}catch(e){}}()==("focus"===t)}function Ee(e,t,n,r,i,o){var a,s;if("object"==typeof t){for(s in"string"!=typeof n&&(r=r||n,n=void 0),t)Ee(e,s,n,r,t[s],o);return e}if(null==r&&null==i?(i=n,r=n=void 0):null==i&&("string"==typeof n?(i=r,r=void 0):(i=r,r=n,n=void 0)),!1===i)i=Te;else if(!i)return e;return 1===o&&(a=i,(i=function(e){return S().off(e),a.apply(this,arguments)}).guid=a.guid||(a.guid=S.guid++)),e.each(function(){S.event.add(this,t,i,r,n)})}function Se(e,i,o){o?(Y.set(e,i,!1),S.event.add(e,i,{namespace:!1,handler:function(e){var t,n,r=Y.get(this,i);if(1&e.isTrigger&&this[i]){if(r.length)(S.event.special[i]||{}).delegateType&&e.stopPropagation();else if(r=s.call(arguments),Y.set(this,i,r),t=o(this,i),this[i](),r!==(n=Y.get(this,i))||t?Y.set(this,i,!1):n={},r!==n)return e.stopImmediatePropagation(),e.preventDefault(),n&&n.value}else r.length&&(Y.set(this,i,{value:S.event.trigger(S.extend(r[0],S.Event.prototype),r.slice(1),this)}),e.stopImmediatePropagation())}})):void 0===Y.get(e,i)&&S.event.add(e,i,we)}S.event={global:{},add:function(t,e,n,r,i){var o,a,s,u,l,c,f,p,d,h,g,v=Y.get(t);if(V(t)){n.handler&&(n=(o=n).handler,i=o.selector),i&&S.find.matchesSelector(re,i),n.guid||(n.guid=S.guid++),(u=v.events)||(u=v.events=Object.create(null)),(a=v.handle)||(a=v.handle=function(e){return"undefined"!=typeof S&&S.event.triggered!==e.type?S.event.dispatch.apply(t,arguments):void 0}),l=(e=(e||"").match(P)||[""]).length;while(l--)d=g=(s=be.exec(e[l])||[])[1],h=(s[2]||"").split(".").sort(),d&&(f=S.event.special[d]||{},d=(i?f.delegateType:f.bindType)||d,f=S.event.special[d]||{},c=S.extend({type:d,origType:g,data:r,handler:n,guid:n.guid,selector:i,needsContext:i&&S.expr.match.needsContext.test(i),namespace:h.join(".")},o),(p=u[d])||((p=u[d]=[]).delegateCount=0,f.setup&&!1!==f.setup.call(t,r,h,a)||t.addEventListener&&t.addEventListener(d,a)),f.add&&(f.add.call(t,c),c.handler.guid||(c.handler.guid=n.guid)),i?p.splice(p.delegateCount++,0,c):p.push(c),S.event.global[d]=!0)}},remove:function(e,t,n,r,i){var o,a,s,u,l,c,f,p,d,h,g,v=Y.hasData(e)&&Y.get(e);if(v&&(u=v.events)){l=(t=(t||"").match(P)||[""]).length;while(l--)if(d=g=(s=be.exec(t[l])||[])[1],h=(s[2]||"").split(".").sort(),d){f=S.event.special[d]||{},p=u[d=(r?f.delegateType:f.bindType)||d]||[],s=s[2]&&new RegExp("(^|\\.)"+h.join("\\.(?:.*\\.|)")+"(\\.|$)"),a=o=p.length;while(o--)c=p[o],!i&&g!==c.origType||n&&n.guid!==c.guid||s&&!s.test(c.namespace)||r&&r!==c.selector&&("**"!==r||!c.selector)||(p.splice(o,1),c.selector&&p.delegateCount--,f.remove&&f.remove.call(e,c));a&&!p.length&&(f.teardown&&!1!==f.teardown.call(e,h,v.handle)||S.removeEvent(e,d,v.handle),delete u[d])}else for(d in u)S.event.remove(e,d+t[l],n,r,!0);S.isEmptyObject(u)&&Y.remove(e,"handle events")}},dispatch:function(e){var t,n,r,i,o,a,s=new Array(arguments.length),u=S.event.fix(e),l=(Y.get(this,"events")||Object.create(null))[u.type]||[],c=S.event.special[u.type]||{};for(s[0]=u,t=1;t<arguments.length;t++)s[t]=arguments[t];if(u.delegateTarget=this,!c.preDispatch||!1!==c.preDispatch.call(this,u)){a=S.event.handlers.call(this,u,l),t=0;while((i=a[t++])&&!u.isPropagationStopped()){u.currentTarget=i.elem,n=0;while((o=i.handlers[n++])&&!u.isImmediatePropagationStopped())u.rnamespace&&!1!==o.namespace&&!u.rnamespace.test(o.namespace)||(u.handleObj=o,u.data=o.data,void 0!==(r=((S.event.special[o.origType]||{}).handle||o.handler).apply(i.elem,s))&&!1===(u.result=r)&&(u.preventDefault(),u.stopPropagation()))}return c.postDispatch&&c.postDispatch.call(this,u),u.result}},handlers:function(e,t){var n,r,i,o,a,s=[],u=t.delegateCount,l=e.target;if(u&&l.nodeType&&!("click"===e.type&&1<=e.button))for(;l!==this;l=l.parentNode||this)if(1===l.nodeType&&("click"!==e.type||!0!==l.disabled)){for(o=[],a={},n=0;n<u;n++)void 0===a[i=(r=t[n]).selector+" "]&&(a[i]=r.needsContext?-1<S(i,this).index(l):S.find(i,this,null,[l]).length),a[i]&&o.push(r);o.length&&s.push({elem:l,handlers:o})}return l=this,u<t.length&&s.push({elem:l,handlers:t.slice(u)}),s},addProp:function(t,e){Object.defineProperty(S.Event.prototype,t,{enumerable:!0,configurable:!0,get:m(e)?function(){if(this.originalEvent)return e(this.originalEvent)}:function(){if(this.originalEvent)return this.originalEvent[t]},set:function(e){Object.defineProperty(this,t,{enumerable:!0,configurable:!0,writable:!0,value:e})}})},fix:function(e){return e[S.expando]?e:new S.Event(e)},special:{load:{noBubble:!0},click:{setup:function(e){var t=this||e;return pe.test(t.type)&&t.click&&A(t,"input")&&Se(t,"click",we),!1},trigger:function(e){var t=this||e;return pe.test(t.type)&&t.click&&A(t,"input")&&Se(t,"click"),!0},_default:function(e){var t=e.target;return pe.test(t.type)&&t.click&&A(t,"input")&&Y.get(t,"click")||A(t,"a")}},beforeunload:{postDispatch:function(e){void 0!==e.result&&e.originalEvent&&(e.originalEvent.returnValue=e.result)}}}},S.removeEvent=function(e,t,n){e.removeEventListener&&e.removeEventListener(t,n)},S.Event=function(e,t){if(!(this instanceof S.Event))return new S.Event(e,t);e&&e.type?(this.originalEvent=e,this.type=e.type,this.isDefaultPrevented=e.defaultPrevented||void 0===e.defaultPrevented&&!1===e.returnValue?we:Te,this.target=e.target&&3===e.target.nodeType?e.target.parentNode:e.target,this.currentTarget=e.currentTarget,this.relatedTarget=e.relatedTarget):this.type=e,t&&S.extend(this,t),this.timeStamp=e&&e.timeStamp||Date.now(),this[S.expando]=!0},S.Event.prototype={constructor:S.Event,isDefaultPrevented:Te,isPropagationStopped:Te,isImmediatePropagationStopped:Te,isSimulated:!1,preventDefault:function(){var e=this.originalEvent;this.isDefaultPrevented=we,e&&!this.isSimulated&&e.preventDefault()},stopPropagation:function(){var e=this.originalEvent;this.isPropagationStopped=we,e&&!this.isSimulated&&e.stopPropagation()},stopImmediatePropagation:function(){var e=this.originalEvent;this.isImmediatePropagationStopped=we,e&&!this.isSimulated&&e.stopImmediatePropagation(),this.stopPropagation()}},S.each({altKey:!0,bubbles:!0,cancelable:!0,changedTouches:!0,ctrlKey:!0,detail:!0,eventPhase:!0,metaKey:!0,pageX:!0,pageY:!0,shiftKey:!0,view:!0,"char":!0,code:!0,charCode:!0,key:!0,keyCode:!0,button:!0,buttons:!0,clientX:!0,clientY:!0,offsetX:!0,offsetY:!0,pointerId:!0,pointerType:!0,screenX:!0,screenY:!0,targetTouches:!0,toElement:!0,touches:!0,which:!0},S.event.addProp),S.each({focus:"focusin",blur:"focusout"},function(e,t){S.event.special[e]={setup:function(){return Se(this,e,Ce),!1},trigger:function(){return Se(this,e),!0},_default:function(){return!0},delegateType:t}}),S.each({mouseenter:"mouseover",mouseleave:"mouseout",pointerenter:"pointerover",pointerleave:"pointerout"},function(e,i){S.event.special[e]={delegateType:i,bindType:i,handle:function(e){var t,n=e.relatedTarget,r=e.handleObj;return n&&(n===this||S.contains(this,n))||(e.type=r.origType,t=r.handler.apply(this,arguments),e.type=i),t}}}),S.fn.extend({on:function(e,t,n,r){return Ee(this,e,t,n,r)},one:function(e,t,n,r){return Ee(this,e,t,n,r,1)},off:function(e,t,n){var r,i;if(e&&e.preventDefault&&e.handleObj)return r=e.handleObj,S(e.delegateTarget).off(r.namespace?r.origType+"."+r.namespace:r.origType,r.selector,r.handler),this;if("object"==typeof e){for(i in e)this.off(i,t,e[i]);return this}return!1!==t&&"function"!=typeof t||(n=t,t=void 0),!1===n&&(n=Te),this.each(function(){S.event.remove(this,e,n,t)})}});var ke=/<script|<style|<link/i,Ae=/checked\s*(?:[^=]|=\s*.checked.)/i,Ne=/^\s*<!(?:\[CDATA\[|--)|(?:\]\]|--)>\s*$/g;function je(e,t){return A(e,"table")&&A(11!==t.nodeType?t:t.firstChild,"tr")&&S(e).children("tbody")[0]||e}function De(e){return e.type=(null!==e.getAttribute("type"))+"/"+e.type,e}function qe(e){return"true/"===(e.type||"").slice(0,5)?e.type=e.type.slice(5):e.removeAttribute("type"),e}function Le(e,t){var n,r,i,o,a,s;if(1===t.nodeType){if(Y.hasData(e)&&(s=Y.get(e).events))for(i in Y.remove(t,"handle events"),s)for(n=0,r=s[i].length;n<r;n++)S.event.add(t,i,s[i][n]);Q.hasData(e)&&(o=Q.access(e),a=S.extend({},o),Q.set(t,a))}}function He(n,r,i,o){r=g(r);var e,t,a,s,u,l,c=0,f=n.length,p=f-1,d=r[0],h=m(d);if(h||1<f&&"string"==typeof d&&!y.checkClone&&Ae.test(d))return n.each(function(e){var t=n.eq(e);h&&(r[0]=d.call(this,e,t.html())),He(t,r,i,o)});if(f&&(t=(e=xe(r,n[0].ownerDocument,!1,n,o)).firstChild,1===e.childNodes.length&&(e=t),t||o)){for(s=(a=S.map(ve(e,"script"),De)).length;c<f;c++)u=e,c!==p&&(u=S.clone(u,!0,!0),s&&S.merge(a,ve(u,"script"))),i.call(n[c],u,c);if(s)for(l=a[a.length-1].ownerDocument,S.map(a,qe),c=0;c<s;c++)u=a[c],he.test(u.type||"")&&!Y.access(u,"globalEval")&&S.contains(l,u)&&(u.src&&"module"!==(u.type||"").toLowerCase()?S._evalUrl&&!u.noModule&&S._evalUrl(u.src,{nonce:u.nonce||u.getAttribute("nonce")},l):b(u.textContent.replace(Ne,""),u,l))}return n}function Oe(e,t,n){for(var r,i=t?S.filter(t,e):e,o=0;null!=(r=i[o]);o++)n||1!==r.nodeType||S.cleanData(ve(r)),r.parentNode&&(n&&ie(r)&&ye(ve(r,"script")),r.parentNode.removeChild(r));return e}S.extend({htmlPrefilter:function(e){return e},clone:function(e,t,n){var r,i,o,a,s,u,l,c=e.cloneNode(!0),f=ie(e);if(!(y.noCloneChecked||1!==e.nodeType&&11!==e.nodeType||S.isXMLDoc(e)))for(a=ve(c),r=0,i=(o=ve(e)).length;r<i;r++)s=o[r],u=a[r],void 0,"input"===(l=u.nodeName.toLowerCase())&&pe.test(s.type)?u.checked=s.checked:"input"!==l&&"textarea"!==l||(u.defaultValue=s.defaultValue);if(t)if(n)for(o=o||ve(e),a=a||ve(c),r=0,i=o.length;r<i;r++)Le(o[r],a[r]);else Le(e,c);return 0<(a=ve(c,"script")).length&&ye(a,!f&&ve(e,"script")),c},cleanData:function(e){for(var t,n,r,i=S.event.special,o=0;void 0!==(n=e[o]);o++)if(V(n)){if(t=n[Y.expando]){if(t.events)for(r in t.events)i[r]?S.event.remove(n,r):S.removeEvent(n,r,t.handle);n[Y.expando]=void 0}n[Q.expando]&&(n[Q.expando]=void 0)}}}),S.fn.extend({detach:function(e){return Oe(this,e,!0)},remove:function(e){return Oe(this,e)},text:function(e){return $(this,function(e){return void 0===e?S.text(this):this.empty().each(function(){1!==this.nodeType&&11!==this.nodeType&&9!==this.nodeType||(this.textContent=e)})},null,e,arguments.length)},append:function(){return He(this,arguments,function(e){1!==this.nodeType&&11!==this.nodeType&&9!==this.nodeType||je(this,e).appendChild(e)})},prepend:function(){return He(this,arguments,function(e){if(1===this.nodeType||11===this.nodeType||9===this.nodeType){var t=je(this,e);t.insertBefore(e,t.firstChild)}})},before:function(){return He(this,arguments,function(e){this.parentNode&&this.parentNode.insertBefore(e,this)})},after:function(){return He(this,arguments,function(e){this.parentNode&&this.parentNode.insertBefore(e,this.nextSibling)})},empty:function(){for(var e,t=0;null!=(e=this[t]);t++)1===e.nodeType&&(S.cleanData(ve(e,!1)),e.textContent="");return this},clone:function(e,t){return e=null!=e&&e,t=null==t?e:t,this.map(function(){return S.clone(this,e,t)})},html:function(e){return $(this,function(e){var t=this[0]||{},n=0,r=this.length;if(void 0===e&&1===t.nodeType)return t.innerHTML;if("string"==typeof e&&!ke.test(e)&&!ge[(de.exec(e)||["",""])[1].toLowerCase()]){e=S.htmlPrefilter(e);try{for(;n<r;n++)1===(t=this[n]||{}).nodeType&&(S.cleanData(ve(t,!1)),t.innerHTML=e);t=0}catch(e){}}t&&this.empty().append(e)},null,e,arguments.length)},replaceWith:function(){var n=[];return He(this,arguments,function(e){var t=this.parentNode;S.inArray(this,n)<0&&(S.cleanData(ve(this)),t&&t.replaceChild(e,this))},n)}}),S.each({appendTo:"append",prependTo:"prepend",insertBefore:"before",insertAfter:"after",replaceAll:"replaceWith"},function(e,a){S.fn[e]=function(e){for(var t,n=[],r=S(e),i=r.length-1,o=0;o<=i;o++)t=o===i?this:this.clone(!0),S(r[o])[a](t),u.apply(n,t.get());return this.pushStack(n)}});var Pe=new RegExp("^("+ee+")(?!px)[a-z%]+$","i"),Re=function(e){var t=e.ownerDocument.defaultView;return t&&t.opener||(t=C),t.getComputedStyle(e)},Me=function(e,t,n){var r,i,o={};for(i in t)o[i]=e.style[i],e.style[i]=t[i];for(i in r=n.call(e),t)e.style[i]=o[i];return r},Ie=new RegExp(ne.join("|"),"i");function We(e,t,n){var r,i,o,a,s=e.style;return(n=n||Re(e))&&(""!==(a=n.getPropertyValue(t)||n[t])||ie(e)||(a=S.style(e,t)),!y.pixelBoxStyles()&&Pe.test(a)&&Ie.test(t)&&(r=s.width,i=s.minWidth,o=s.maxWidth,s.minWidth=s.maxWidth=s.width=a,a=n.width,s.width=r,s.minWidth=i,s.maxWidth=o)),void 0!==a?a+"":a}function Fe(e,t){return{get:function(){if(!e())return(this.get=t).apply(this,arguments);delete this.get}}}!function(){function e(){if(l){u.style.cssText="position:absolute;left:-11111px;width:60px;margin-top:1px;padding:0;border:0",l.style.cssText="position:relative;display:block;box-sizing:border-box;overflow:scroll;margin:auto;border:1px;padding:1px;width:60%;top:1%",re.appendChild(u).appendChild(l);var e=C.getComputedStyle(l);n="1%"!==e.top,s=12===t(e.marginLeft),l.style.right="60%",o=36===t(e.right),r=36===t(e.width),l.style.position="absolute",i=12===t(l.offsetWidth/3),re.removeChild(u),l=null}}function t(e){return Math.round(parseFloat(e))}var n,r,i,o,a,s,u=E.createElement("div"),l=E.createElement("div");l.style&&(l.style.backgroundClip="content-box",l.cloneNode(!0).style.backgroundClip="",y.clearCloneStyle="content-box"===l.style.backgroundClip,S.extend(y,{boxSizingReliable:function(){return e(),r},pixelBoxStyles:function(){return e(),o},pixelPosition:function(){return e(),n},reliableMarginLeft:function(){return e(),s},scrollboxSize:function(){return e(),i},reliableTrDimensions:function(){var e,t,n,r;return null==a&&(e=E.createElement("table"),t=E.createElement("tr"),n=E.createElement("div"),e.style.cssText="position:absolute;left:-11111px;border-collapse:separate",t.style.cssText="border:1px solid",t.style.height="1px",n.style.height="9px",n.style.display="block",re.appendChild(e).appendChild(t).appendChild(n),r=C.getComputedStyle(t),a=parseInt(r.height,10)+parseInt(r.borderTopWidth,10)+parseInt(r.borderBottomWidth,10)===t.offsetHeight,re.removeChild(e)),a}}))}();var Be=["Webkit","Moz","ms"],$e=E.createElement("div").style,_e={};function ze(e){var t=S.cssProps[e]||_e[e];return t||(e in $e?e:_e[e]=function(e){var t=e[0].toUpperCase()+e.slice(1),n=Be.length;while(n--)if((e=Be[n]+t)in $e)return e}(e)||e)}var Ue=/^(none|table(?!-c[ea]).+)/,Xe=/^--/,Ve={position:"absolute",visibility:"hidden",display:"block"},Ge={letterSpacing:"0",fontWeight:"400"};function Ye(e,t,n){var r=te.exec(t);return r?Math.max(0,r[2]-(n||0))+(r[3]||"px"):t}function Qe(e,t,n,r,i,o){var a="width"===t?1:0,s=0,u=0;if(n===(r?"border":"content"))return 0;for(;a<4;a+=2)"margin"===n&&(u+=S.css(e,n+ne[a],!0,i)),r?("content"===n&&(u-=S.css(e,"padding"+ne[a],!0,i)),"margin"!==n&&(u-=S.css(e,"border"+ne[a]+"Width",!0,i))):(u+=S.css(e,"padding"+ne[a],!0,i),"padding"!==n?u+=S.css(e,"border"+ne[a]+"Width",!0,i):s+=S.css(e,"border"+ne[a]+"Width",!0,i));return!r&&0<=o&&(u+=Math.max(0,Math.ceil(e["offset"+t[0].toUpperCase()+t.slice(1)]-o-u-s-.5))||0),u}function Je(e,t,n){var r=Re(e),i=(!y.boxSizingReliable()||n)&&"border-box"===S.css(e,"boxSizing",!1,r),o=i,a=We(e,t,r),s="offset"+t[0].toUpperCase()+t.slice(1);if(Pe.test(a)){if(!n)return a;a="auto"}return(!y.boxSizingReliable()&&i||!y.reliableTrDimensions()&&A(e,"tr")||"auto"===a||!parseFloat(a)&&"inline"===S.css(e,"display",!1,r))&&e.getClientRects().length&&(i="border-box"===S.css(e,"boxSizing",!1,r),(o=s in e)&&(a=e[s])),(a=parseFloat(a)||0)+Qe(e,t,n||(i?"border":"content"),o,r,a)+"px"}function Ke(e,t,n,r,i){return new Ke.prototype.init(e,t,n,r,i)}S.extend({cssHooks:{opacity:{get:function(e,t){if(t){var n=We(e,"opacity");return""===n?"1":n}}}},cssNumber:{animationIterationCount:!0,columnCount:!0,fillOpacity:!0,flexGrow:!0,flexShrink:!0,fontWeight:!0,gridArea:!0,gridColumn:!0,gridColumnEnd:!0,gridColumnStart:!0,gridRow:!0,gridRowEnd:!0,gridRowStart:!0,lineHeight:!0,opacity:!0,order:!0,orphans:!0,widows:!0,zIndex:!0,zoom:!0},cssProps:{},style:function(e,t,n,r){if(e&&3!==e.nodeType&&8!==e.nodeType&&e.style){var i,o,a,s=X(t),u=Xe.test(t),l=e.style;if(u||(t=ze(s)),a=S.cssHooks[t]||S.cssHooks[s],void 0===n)return a&&"get"in a&&void 0!==(i=a.get(e,!1,r))?i:l[t];"string"===(o=typeof n)&&(i=te.exec(n))&&i[1]&&(n=se(e,t,i),o="number"),null!=n&&n==n&&("number"!==o||u||(n+=i&&i[3]||(S.cssNumber[s]?"":"px")),y.clearCloneStyle||""!==n||0!==t.indexOf("background")||(l[t]="inherit"),a&&"set"in a&&void 0===(n=a.set(e,n,r))||(u?l.setProperty(t,n):l[t]=n))}},css:function(e,t,n,r){var i,o,a,s=X(t);return Xe.test(t)||(t=ze(s)),(a=S.cssHooks[t]||S.cssHooks[s])&&"get"in a&&(i=a.get(e,!0,n)),void 0===i&&(i=We(e,t,r)),"normal"===i&&t in Ge&&(i=Ge[t]),""===n||n?(o=parseFloat(i),!0===n||isFinite(o)?o||0:i):i}}),S.each(["height","width"],function(e,u){S.cssHooks[u]={get:function(e,t,n){if(t)return!Ue.test(S.css(e,"display"))||e.getClientRects().length&&e.getBoundingClientRect().width?Je(e,u,n):Me(e,Ve,function(){return Je(e,u,n)})},set:function(e,t,n){var r,i=Re(e),o=!y.scrollboxSize()&&"absolute"===i.position,a=(o||n)&&"border-box"===S.css(e,"boxSizing",!1,i),s=n?Qe(e,u,n,a,i):0;return a&&o&&(s-=Math.ceil(e["offset"+u[0].toUpperCase()+u.slice(1)]-parseFloat(i[u])-Qe(e,u,"border",!1,i)-.5)),s&&(r=te.exec(t))&&"px"!==(r[3]||"px")&&(e.style[u]=t,t=S.css(e,u)),Ye(0,t,s)}}}),S.cssHooks.marginLeft=Fe(y.reliableMarginLeft,function(e,t){if(t)return(parseFloat(We(e,"marginLeft"))||e.getBoundingClientRect().left-Me(e,{marginLeft:0},function(){return e.getBoundingClientRect().left}))+"px"}),S.each({margin:"",padding:"",border:"Width"},function(i,o){S.cssHooks[i+o]={expand:function(e){for(var t=0,n={},r="string"==typeof e?e.split(" "):[e];t<4;t++)n[i+ne[t]+o]=r[t]||r[t-2]||r[0];return n}},"margin"!==i&&(S.cssHooks[i+o].set=Ye)}),S.fn.extend({css:function(e,t){return $(this,function(e,t,n){var r,i,o={},a=0;if(Array.isArray(t)){for(r=Re(e),i=t.length;a<i;a++)o[t[a]]=S.css(e,t[a],!1,r);return o}return void 0!==n?S.style(e,t,n):S.css(e,t)},e,t,1<arguments.length)}}),((S.Tween=Ke).prototype={constructor:Ke,init:function(e,t,n,r,i,o){this.elem=e,this.prop=n,this.easing=i||S.easing._default,this.options=t,this.start=this.now=this.cur(),this.end=r,this.unit=o||(S.cssNumber[n]?"":"px")},cur:function(){var e=Ke.propHooks[this.prop];return e&&e.get?e.get(this):Ke.propHooks._default.get(this)},run:function(e){var t,n=Ke.propHooks[this.prop];return this.options.duration?this.pos=t=S.easing[this.easing](e,this.options.duration*e,0,1,this.options.duration):this.pos=t=e,this.now=(this.end-this.start)*t+this.start,this.options.step&&this.options.step.call(this.elem,this.now,this),n&&n.set?n.set(this):Ke.propHooks._default.set(this),this}}).init.prototype=Ke.prototype,(Ke.propHooks={_default:{get:function(e){var t;return 1!==e.elem.nodeType||null!=e.elem[e.prop]&&null==e.elem.style[e.prop]?e.elem[e.prop]:(t=S.css(e.elem,e.prop,""))&&"auto"!==t?t:0},set:function(e){S.fx.step[e.prop]?S.fx.step[e.prop](e):1!==e.elem.nodeType||!S.cssHooks[e.prop]&&null==e.elem.style[ze(e.prop)]?e.elem[e.prop]=e.now:S.style(e.elem,e.prop,e.now+e.unit)}}}).scrollTop=Ke.propHooks.scrollLeft={set:function(e){e.elem.nodeType&&e.elem.parentNode&&(e.elem[e.prop]=e.now)}},S.easing={linear:function(e){return e},swing:function(e){return.5-Math.cos(e*Math.PI)/2},_default:"swing"},S.fx=Ke.prototype.init,S.fx.step={};var Ze,et,tt,nt,rt=/^(?:toggle|show|hide)$/,it=/queueHooks$/;function ot(){et&&(!1===E.hidden&&C.requestAnimationFrame?C.requestAnimationFrame(ot):C.setTimeout(ot,S.fx.interval),S.fx.tick())}function at(){return C.setTimeout(function(){Ze=void 0}),Ze=Date.now()}function st(e,t){var n,r=0,i={height:e};for(t=t?1:0;r<4;r+=2-t)i["margin"+(n=ne[r])]=i["padding"+n]=e;return t&&(i.opacity=i.width=e),i}function ut(e,t,n){for(var r,i=(lt.tweeners[t]||[]).concat(lt.tweeners["*"]),o=0,a=i.length;o<a;o++)if(r=i[o].call(n,t,e))return r}function lt(o,e,t){var n,a,r=0,i=lt.prefilters.length,s=S.Deferred().always(function(){delete u.elem}),u=function(){if(a)return!1;for(var e=Ze||at(),t=Math.max(0,l.startTime+l.duration-e),n=1-(t/l.duration||0),r=0,i=l.tweens.length;r<i;r++)l.tweens[r].run(n);return s.notifyWith(o,[l,n,t]),n<1&&i?t:(i||s.notifyWith(o,[l,1,0]),s.resolveWith(o,[l]),!1)},l=s.promise({elem:o,props:S.extend({},e),opts:S.extend(!0,{specialEasing:{},easing:S.easing._default},t),originalProperties:e,originalOptions:t,startTime:Ze||at(),duration:t.duration,tweens:[],createTween:function(e,t){var n=S.Tween(o,l.opts,e,t,l.opts.specialEasing[e]||l.opts.easing);return l.tweens.push(n),n},stop:function(e){var t=0,n=e?l.tweens.length:0;if(a)return this;for(a=!0;t<n;t++)l.tweens[t].run(1);return e?(s.notifyWith(o,[l,1,0]),s.resolveWith(o,[l,e])):s.rejectWith(o,[l,e]),this}}),c=l.props;for(!function(e,t){var n,r,i,o,a;for(n in e)if(i=t[r=X(n)],o=e[n],Array.isArray(o)&&(i=o[1],o=e[n]=o[0]),n!==r&&(e[r]=o,delete e[n]),(a=S.cssHooks[r])&&"expand"in a)for(n in o=a.expand(o),delete e[r],o)n in e||(e[n]=o[n],t[n]=i);else t[r]=i}(c,l.opts.specialEasing);r<i;r++)if(n=lt.prefilters[r].call(l,o,c,l.opts))return m(n.stop)&&(S._queueHooks(l.elem,l.opts.queue).stop=n.stop.bind(n)),n;return S.map(c,ut,l),m(l.opts.start)&&l.opts.start.call(o,l),l.progress(l.opts.progress).done(l.opts.done,l.opts.complete).fail(l.opts.fail).always(l.opts.always),S.fx.timer(S.extend(u,{elem:o,anim:l,queue:l.opts.queue})),l}S.Animation=S.extend(lt,{tweeners:{"*":[function(e,t){var n=this.createTween(e,t);return se(n.elem,e,te.exec(t),n),n}]},tweener:function(e,t){m(e)?(t=e,e=["*"]):e=e.match(P);for(var n,r=0,i=e.length;r<i;r++)n=e[r],lt.tweeners[n]=lt.tweeners[n]||[],lt.tweeners[n].unshift(t)},prefilters:[function(e,t,n){var r,i,o,a,s,u,l,c,f="width"in t||"height"in t,p=this,d={},h=e.style,g=e.nodeType&&ae(e),v=Y.get(e,"fxshow");for(r in n.queue||(null==(a=S._queueHooks(e,"fx")).unqueued&&(a.unqueued=0,s=a.empty.fire,a.empty.fire=function(){a.unqueued||s()}),a.unqueued++,p.always(function(){p.always(function(){a.unqueued--,S.queue(e,"fx").length||a.empty.fire()})})),t)if(i=t[r],rt.test(i)){if(delete t[r],o=o||"toggle"===i,i===(g?"hide":"show")){if("show"!==i||!v||void 0===v[r])continue;g=!0}d[r]=v&&v[r]||S.style(e,r)}if((u=!S.isEmptyObject(t))||!S.isEmptyObject(d))for(r in f&&1===e.nodeType&&(n.overflow=[h.overflow,h.overflowX,h.overflowY],null==(l=v&&v.display)&&(l=Y.get(e,"display")),"none"===(c=S.css(e,"display"))&&(l?c=l:(le([e],!0),l=e.style.display||l,c=S.css(e,"display"),le([e]))),("inline"===c||"inline-block"===c&&null!=l)&&"none"===S.css(e,"float")&&(u||(p.done(function(){h.display=l}),null==l&&(c=h.display,l="none"===c?"":c)),h.display="inline-block")),n.overflow&&(h.overflow="hidden",p.always(function(){h.overflow=n.overflow[0],h.overflowX=n.overflow[1],h.overflowY=n.overflow[2]})),u=!1,d)u||(v?"hidden"in v&&(g=v.hidden):v=Y.access(e,"fxshow",{display:l}),o&&(v.hidden=!g),g&&le([e],!0),p.done(function(){for(r in g||le([e]),Y.remove(e,"fxshow"),d)S.style(e,r,d[r])})),u=ut(g?v[r]:0,r,p),r in v||(v[r]=u.start,g&&(u.end=u.start,u.start=0))}],prefilter:function(e,t){t?lt.prefilters.unshift(e):lt.prefilters.push(e)}}),S.speed=function(e,t,n){var r=e&&"object"==typeof e?S.extend({},e):{complete:n||!n&&t||m(e)&&e,duration:e,easing:n&&t||t&&!m(t)&&t};return S.fx.off?r.duration=0:"number"!=typeof r.duration&&(r.duration in S.fx.speeds?r.duration=S.fx.speeds[r.duration]:r.duration=S.fx.speeds._default),null!=r.queue&&!0!==r.queue||(r.queue="fx"),r.old=r.complete,r.complete=function(){m(r.old)&&r.old.call(this),r.queue&&S.dequeue(this,r.queue)},r},S.fn.extend({fadeTo:function(e,t,n,r){return this.filter(ae).css("opacity",0).show().end().animate({opacity:t},e,n,r)},animate:function(t,e,n,r){var i=S.isEmptyObject(t),o=S.speed(e,n,r),a=function(){var e=lt(this,S.extend({},t),o);(i||Y.get(this,"finish"))&&e.stop(!0)};return a.finish=a,i||!1===o.queue?this.each(a):this.queue(o.queue,a)},stop:function(i,e,o){var a=function(e){var t=e.stop;delete e.stop,t(o)};return"string"!=typeof i&&(o=e,e=i,i=void 0),e&&this.queue(i||"fx",[]),this.each(function(){var e=!0,t=null!=i&&i+"queueHooks",n=S.timers,r=Y.get(this);if(t)r[t]&&r[t].stop&&a(r[t]);else for(t in r)r[t]&&r[t].stop&&it.test(t)&&a(r[t]);for(t=n.length;t--;)n[t].elem!==this||null!=i&&n[t].queue!==i||(n[t].anim.stop(o),e=!1,n.splice(t,1));!e&&o||S.dequeue(this,i)})},finish:function(a){return!1!==a&&(a=a||"fx"),this.each(function(){var e,t=Y.get(this),n=t[a+"queue"],r=t[a+"queueHooks"],i=S.timers,o=n?n.length:0;for(t.finish=!0,S.queue(this,a,[]),r&&r.stop&&r.stop.call(this,!0),e=i.length;e--;)i[e].elem===this&&i[e].queue===a&&(i[e].anim.stop(!0),i.splice(e,1));for(e=0;e<o;e++)n[e]&&n[e].finish&&n[e].finish.call(this);delete t.finish})}}),S.each(["toggle","show","hide"],function(e,r){var i=S.fn[r];S.fn[r]=function(e,t,n){return null==e||"boolean"==typeof e?i.apply(this,arguments):this.animate(st(r,!0),e,t,n)}}),S.each({slideDown:st("show"),slideUp:st("hide"),slideToggle:st("toggle"),fadeIn:{opacity:"show"},fadeOut:{opacity:"hide"},fadeToggle:{opacity:"toggle"}},function(e,r){S.fn[e]=function(e,t,n){return this.animate(r,e,t,n)}}),S.timers=[],S.fx.tick=function(){var e,t=0,n=S.timers;for(Ze=Date.now();t<n.length;t++)(e=n[t])()||n[t]!==e||n.splice(t--,1);n.length||S.fx.stop(),Ze=void 0},S.fx.timer=function(e){S.timers.push(e),S.fx.start()},S.fx.interval=13,S.fx.start=function(){et||(et=!0,ot())},S.fx.stop=function(){et=null},S.fx.speeds={slow:600,fast:200,_default:400},S.fn.delay=function(r,e){return r=S.fx&&S.fx.speeds[r]||r,e=e||"fx",this.queue(e,function(e,t){var n=C.setTimeout(e,r);t.stop=function(){C.clearTimeout(n)}})},tt=E.createElement("input"),nt=E.createElement("select").appendChild(E.createElement("option")),tt.type="checkbox",y.checkOn=""!==tt.value,y.optSelected=nt.selected,(tt=E.createElement("input")).value="t",tt.type="radio",y.radioValue="t"===tt.value;var ct,ft=S.expr.attrHandle;S.fn.extend({attr:function(e,t){return $(this,S.attr,e,t,1<arguments.length)},removeAttr:function(e){return this.each(function(){S.removeAttr(this,e)})}}),S.extend({attr:function(e,t,n){var r,i,o=e.nodeType;if(3!==o&&8!==o&&2!==o)return"undefined"==typeof e.getAttribute?S.prop(e,t,n):(1===o&&S.isXMLDoc(e)||(i=S.attrHooks[t.toLowerCase()]||(S.expr.match.bool.test(t)?ct:void 0)),void 0!==n?null===n?void S.removeAttr(e,t):i&&"set"in i&&void 0!==(r=i.set(e,n,t))?r:(e.setAttribute(t,n+""),n):i&&"get"in i&&null!==(r=i.get(e,t))?r:null==(r=S.find.attr(e,t))?void 0:r)},attrHooks:{type:{set:function(e,t){if(!y.radioValue&&"radio"===t&&A(e,"input")){var n=e.value;return e.setAttribute("type",t),n&&(e.value=n),t}}}},removeAttr:function(e,t){var n,r=0,i=t&&t.match(P);if(i&&1===e.nodeType)while(n=i[r++])e.removeAttribute(n)}}),ct={set:function(e,t,n){return!1===t?S.removeAttr(e,n):e.setAttribute(n,n),n}},S.each(S.expr.match.bool.source.match(/\w+/g),function(e,t){var a=ft[t]||S.find.attr;ft[t]=function(e,t,n){var r,i,o=t.toLowerCase();return n||(i=ft[o],ft[o]=r,r=null!=a(e,t,n)?o:null,ft[o]=i),r}});var pt=/^(?:input|select|textarea|button)$/i,dt=/^(?:a|area)$/i;function ht(e){return(e.match(P)||[]).join(" ")}function gt(e){return e.getAttribute&&e.getAttribute("class")||""}function vt(e){return Array.isArray(e)?e:"string"==typeof e&&e.match(P)||[]}S.fn.extend({prop:function(e,t){return $(this,S.prop,e,t,1<arguments.length)},removeProp:function(e){return this.each(function(){delete this[S.propFix[e]||e]})}}),S.extend({prop:function(e,t,n){var r,i,o=e.nodeType;if(3!==o&&8!==o&&2!==o)return 1===o&&S.isXMLDoc(e)||(t=S.propFix[t]||t,i=S.propHooks[t]),void 0!==n?i&&"set"in i&&void 0!==(r=i.set(e,n,t))?r:e[t]=n:i&&"get"in i&&null!==(r=i.get(e,t))?r:e[t]},propHooks:{tabIndex:{get:function(e){var t=S.find.attr(e,"tabindex");return t?parseInt(t,10):pt.test(e.nodeName)||dt.test(e.nodeName)&&e.href?0:-1}}},propFix:{"for":"htmlFor","class":"className"}}),y.optSelected||(S.propHooks.selected={get:function(e){var t=e.parentNode;return t&&t.parentNode&&t.parentNode.selectedIndex,null},set:function(e){var t=e.parentNode;t&&(t.selectedIndex,t.parentNode&&t.parentNode.selectedIndex)}}),S.each(["tabIndex","readOnly","maxLength","cellSpacing","cellPadding","rowSpan","colSpan","useMap","frameBorder","contentEditable"],function(){S.propFix[this.toLowerCase()]=this}),S.fn.extend({addClass:function(t){var e,n,r,i,o,a,s,u=0;if(m(t))return this.each(function(e){S(this).addClass(t.call(this,e,gt(this)))});if((e=vt(t)).length)while(n=this[u++])if(i=gt(n),r=1===n.nodeType&&" "+ht(i)+" "){a=0;while(o=e[a++])r.indexOf(" "+o+" ")<0&&(r+=o+" ");i!==(s=ht(r))&&n.setAttribute("class",s)}return this},removeClass:function(t){var e,n,r,i,o,a,s,u=0;if(m(t))return this.each(function(e){S(this).removeClass(t.call(this,e,gt(this)))});if(!arguments.length)return this.attr("class","");if((e=vt(t)).length)while(n=this[u++])if(i=gt(n),r=1===n.nodeType&&" "+ht(i)+" "){a=0;while(o=e[a++])while(-1<r.indexOf(" "+o+" "))r=r.replace(" "+o+" "," ");i!==(s=ht(r))&&n.setAttribute("class",s)}return this},toggleClass:function(i,t){var o=typeof i,a="string"===o||Array.isArray(i);return"boolean"==typeof t&&a?t?this.addClass(i):this.removeClass(i):m(i)?this.each(function(e){S(this).toggleClass(i.call(this,e,gt(this),t),t)}):this.each(function(){var e,t,n,r;if(a){t=0,n=S(this),r=vt(i);while(e=r[t++])n.hasClass(e)?n.removeClass(e):n.addClass(e)}else void 0!==i&&"boolean"!==o||((e=gt(this))&&Y.set(this,"__className__",e),this.setAttribute&&this.setAttribute("class",e||!1===i?"":Y.get(this,"__className__")||""))})},hasClass:function(e){var t,n,r=0;t=" "+e+" ";while(n=this[r++])if(1===n.nodeType&&-1<(" "+ht(gt(n))+" ").indexOf(t))return!0;return!1}});var yt=/\r/g;S.fn.extend({val:function(n){var r,e,i,t=this[0];return arguments.length?(i=m(n),this.each(function(e){var t;1===this.nodeType&&(null==(t=i?n.call(this,e,S(this).val()):n)?t="":"number"==typeof t?t+="":Array.isArray(t)&&(t=S.map(t,function(e){return null==e?"":e+""})),(r=S.valHooks[this.type]||S.valHooks[this.nodeName.toLowerCase()])&&"set"in r&&void 0!==r.set(this,t,"value")||(this.value=t))})):t?(r=S.valHooks[t.type]||S.valHooks[t.nodeName.toLowerCase()])&&"get"in r&&void 0!==(e=r.get(t,"value"))?e:"string"==typeof(e=t.value)?e.replace(yt,""):null==e?"":e:void 0}}),S.extend({valHooks:{option:{get:function(e){var t=S.find.attr(e,"value");return null!=t?t:ht(S.text(e))}},select:{get:function(e){var t,n,r,i=e.options,o=e.selectedIndex,a="select-one"===e.type,s=a?null:[],u=a?o+1:i.length;for(r=o<0?u:a?o:0;r<u;r++)if(((n=i[r]).selected||r===o)&&!n.disabled&&(!n.parentNode.disabled||!A(n.parentNode,"optgroup"))){if(t=S(n).val(),a)return t;s.push(t)}return s},set:function(e,t){var n,r,i=e.options,o=S.makeArray(t),a=i.length;while(a--)((r=i[a]).selected=-1<S.inArray(S.valHooks.option.get(r),o))&&(n=!0);return n||(e.selectedIndex=-1),o}}}}),S.each(["radio","checkbox"],function(){S.valHooks[this]={set:function(e,t){if(Array.isArray(t))return e.checked=-1<S.inArray(S(e).val(),t)}},y.checkOn||(S.valHooks[this].get=function(e){return null===e.getAttribute("value")?"on":e.value})}),y.focusin="onfocusin"in C;var mt=/^(?:focusinfocus|focusoutblur)$/,xt=function(e){e.stopPropagation()};S.extend(S.event,{trigger:function(e,t,n,r){var i,o,a,s,u,l,c,f,p=[n||E],d=v.call(e,"type")?e.type:e,h=v.call(e,"namespace")?e.namespace.split("."):[];if(o=f=a=n=n||E,3!==n.nodeType&&8!==n.nodeType&&!mt.test(d+S.event.triggered)&&(-1<d.indexOf(".")&&(d=(h=d.split(".")).shift(),h.sort()),u=d.indexOf(":")<0&&"on"+d,(e=e[S.expando]?e:new S.Event(d,"object"==typeof e&&e)).isTrigger=r?2:3,e.namespace=h.join("."),e.rnamespace=e.namespace?new RegExp("(^|\\.)"+h.join("\\.(?:.*\\.|)")+"(\\.|$)"):null,e.result=void 0,e.target||(e.target=n),t=null==t?[e]:S.makeArray(t,[e]),c=S.event.special[d]||{},r||!c.trigger||!1!==c.trigger.apply(n,t))){if(!r&&!c.noBubble&&!x(n)){for(s=c.delegateType||d,mt.test(s+d)||(o=o.parentNode);o;o=o.parentNode)p.push(o),a=o;a===(n.ownerDocument||E)&&p.push(a.defaultView||a.parentWindow||C)}i=0;while((o=p[i++])&&!e.isPropagationStopped())f=o,e.type=1<i?s:c.bindType||d,(l=(Y.get(o,"events")||Object.create(null))[e.type]&&Y.get(o,"handle"))&&l.apply(o,t),(l=u&&o[u])&&l.apply&&V(o)&&(e.result=l.apply(o,t),!1===e.result&&e.preventDefault());return e.type=d,r||e.isDefaultPrevented()||c._default&&!1!==c._default.apply(p.pop(),t)||!V(n)||u&&m(n[d])&&!x(n)&&((a=n[u])&&(n[u]=null),S.event.triggered=d,e.isPropagationStopped()&&f.addEventListener(d,xt),n[d](),e.isPropagationStopped()&&f.removeEventListener(d,xt),S.event.triggered=void 0,a&&(n[u]=a)),e.result}},simulate:function(e,t,n){var r=S.extend(new S.Event,n,{type:e,isSimulated:!0});S.event.trigger(r,null,t)}}),S.fn.extend({trigger:function(e,t){return this.each(function(){S.event.trigger(e,t,this)})},triggerHandler:function(e,t){var n=this[0];if(n)return S.event.trigger(e,t,n,!0)}}),y.focusin||S.each({focus:"focusin",blur:"focusout"},function(n,r){var i=function(e){S.event.simulate(r,e.target,S.event.fix(e))};S.event.special[r]={setup:function(){var e=this.ownerDocument||this.document||this,t=Y.access(e,r);t||e.addEventListener(n,i,!0),Y.access(e,r,(t||0)+1)},teardown:function(){var e=this.ownerDocument||this.document||this,t=Y.access(e,r)-1;t?Y.access(e,r,t):(e.removeEventListener(n,i,!0),Y.remove(e,r))}}});var bt=C.location,wt={guid:Date.now()},Tt=/\?/;S.parseXML=function(e){var t,n;if(!e||"string"!=typeof e)return null;try{t=(new C.DOMParser).parseFromString(e,"text/xml")}catch(e){}return n=t&&t.getElementsByTagName("parsererror")[0],t&&!n||S.error("Invalid XML: "+(n?S.map(n.childNodes,function(e){return e.textContent}).join("\n"):e)),t};var Ct=/\[\]$/,Et=/\r?\n/g,St=/^(?:submit|button|image|reset|file)$/i,kt=/^(?:input|select|textarea|keygen)/i;function At(n,e,r,i){var t;if(Array.isArray(e))S.each(e,function(e,t){r||Ct.test(n)?i(n,t):At(n+"["+("object"==typeof t&&null!=t?e:"")+"]",t,r,i)});else if(r||"object"!==w(e))i(n,e);else for(t in e)At(n+"["+t+"]",e[t],r,i)}S.param=function(e,t){var n,r=[],i=function(e,t){var n=m(t)?t():t;r[r.length]=encodeURIComponent(e)+"="+encodeURIComponent(null==n?"":n)};if(null==e)return"";if(Array.isArray(e)||e.jquery&&!S.isPlainObject(e))S.each(e,function(){i(this.name,this.value)});else for(n in e)At(n,e[n],t,i);return r.join("&")},S.fn.extend({serialize:function(){return S.param(this.serializeArray())},serializeArray:function(){return this.map(function(){var e=S.prop(this,"elements");return e?S.makeArray(e):this}).filter(function(){var e=this.type;return this.name&&!S(this).is(":disabled")&&kt.test(this.nodeName)&&!St.test(e)&&(this.checked||!pe.test(e))}).map(function(e,t){var n=S(this).val();return null==n?null:Array.isArray(n)?S.map(n,function(e){return{name:t.name,value:e.replace(Et,"\r\n")}}):{name:t.name,value:n.replace(Et,"\r\n")}}).get()}});var Nt=/%20/g,jt=/#.*$/,Dt=/([?&])_=[^&]*/,qt=/^(.*?):[ \t]*([^\r\n]*)$/gm,Lt=/^(?:GET|HEAD)$/,Ht=/^\/\//,Ot={},Pt={},Rt="*/".concat("*"),Mt=E.createElement("a");function It(o){return function(e,t){"string"!=typeof e&&(t=e,e="*");var n,r=0,i=e.toLowerCase().match(P)||[];if(m(t))while(n=i[r++])"+"===n[0]?(n=n.slice(1)||"*",(o[n]=o[n]||[]).unshift(t)):(o[n]=o[n]||[]).push(t)}}function Wt(t,i,o,a){var s={},u=t===Pt;function l(e){var r;return s[e]=!0,S.each(t[e]||[],function(e,t){var n=t(i,o,a);return"string"!=typeof n||u||s[n]?u?!(r=n):void 0:(i.dataTypes.unshift(n),l(n),!1)}),r}return l(i.dataTypes[0])||!s["*"]&&l("*")}function Ft(e,t){var n,r,i=S.ajaxSettings.flatOptions||{};for(n in t)void 0!==t[n]&&((i[n]?e:r||(r={}))[n]=t[n]);return r&&S.extend(!0,e,r),e}Mt.href=bt.href,S.extend({active:0,lastModified:{},etag:{},ajaxSettings:{url:bt.href,type:"GET",isLocal:/^(?:about|app|app-storage|.+-extension|file|res|widget):$/.test(bt.protocol),global:!0,processData:!0,async:!0,contentType:"application/x-www-form-urlencoded; charset=UTF-8",accepts:{"*":Rt,text:"text/plain",html:"text/html",xml:"application/xml, text/xml",json:"application/json, text/javascript"},contents:{xml:/\bxml\b/,html:/\bhtml/,json:/\bjson\b/},responseFields:{xml:"responseXML",text:"responseText",json:"responseJSON"},converters:{"* text":String,"text html":!0,"text json":JSON.parse,"text xml":S.parseXML},flatOptions:{url:!0,context:!0}},ajaxSetup:function(e,t){return t?Ft(Ft(e,S.ajaxSettings),t):Ft(S.ajaxSettings,e)},ajaxPrefilter:It(Ot),ajaxTransport:It(Pt),ajax:function(e,t){"object"==typeof e&&(t=e,e=void 0),t=t||{};var c,f,p,n,d,r,h,g,i,o,v=S.ajaxSetup({},t),y=v.context||v,m=v.context&&(y.nodeType||y.jquery)?S(y):S.event,x=S.Deferred(),b=S.Callbacks("once memory"),w=v.statusCode||{},a={},s={},u="canceled",T={readyState:0,getResponseHeader:function(e){var t;if(h){if(!n){n={};while(t=qt.exec(p))n[t[1].toLowerCase()+" "]=(n[t[1].toLowerCase()+" "]||[]).concat(t[2])}t=n[e.toLowerCase()+" "]}return null==t?null:t.join(", ")},getAllResponseHeaders:function(){return h?p:null},setRequestHeader:function(e,t){return null==h&&(e=s[e.toLowerCase()]=s[e.toLowerCase()]||e,a[e]=t),this},overrideMimeType:function(e){return null==h&&(v.mimeType=e),this},statusCode:function(e){var t;if(e)if(h)T.always(e[T.status]);else for(t in e)w[t]=[w[t],e[t]];return this},abort:function(e){var t=e||u;return c&&c.abort(t),l(0,t),this}};if(x.promise(T),v.url=((e||v.url||bt.href)+"").replace(Ht,bt.protocol+"//"),v.type=t.method||t.type||v.method||v.type,v.dataTypes=(v.dataType||"*").toLowerCase().match(P)||[""],null==v.crossDomain){r=E.createElement("a");try{r.href=v.url,r.href=r.href,v.crossDomain=Mt.protocol+"//"+Mt.host!=r.protocol+"//"+r.host}catch(e){v.crossDomain=!0}}if(v.data&&v.processData&&"string"!=typeof v.data&&(v.data=S.param(v.data,v.traditional)),Wt(Ot,v,t,T),h)return T;for(i in(g=S.event&&v.global)&&0==S.active++&&S.event.trigger("ajaxStart"),v.type=v.type.toUpperCase(),v.hasContent=!Lt.test(v.type),f=v.url.replace(jt,""),v.hasContent?v.data&&v.processData&&0===(v.contentType||"").indexOf("application/x-www-form-urlencoded")&&(v.data=v.data.replace(Nt,"+")):(o=v.url.slice(f.length),v.data&&(v.processData||"string"==typeof v.data)&&(f+=(Tt.test(f)?"&":"?")+v.data,delete v.data),!1===v.cache&&(f=f.replace(Dt,"$1"),o=(Tt.test(f)?"&":"?")+"_="+wt.guid+++o),v.url=f+o),v.ifModified&&(S.lastModified[f]&&T.setRequestHeader("If-Modified-Since",S.lastModified[f]),S.etag[f]&&T.setRequestHeader("If-None-Match",S.etag[f])),(v.data&&v.hasContent&&!1!==v.contentType||t.contentType)&&T.setRequestHeader("Content-Type",v.contentType),T.setRequestHeader("Accept",v.dataTypes[0]&&v.accepts[v.dataTypes[0]]?v.accepts[v.dataTypes[0]]+("*"!==v.dataTypes[0]?", "+Rt+"; q=0.01":""):v.accepts["*"]),v.headers)T.setRequestHeader(i,v.headers[i]);if(v.beforeSend&&(!1===v.beforeSend.call(y,T,v)||h))return T.abort();if(u="abort",b.add(v.complete),T.done(v.success),T.fail(v.error),c=Wt(Pt,v,t,T)){if(T.readyState=1,g&&m.trigger("ajaxSend",[T,v]),h)return T;v.async&&0<v.timeout&&(d=C.setTimeout(function(){T.abort("timeout")},v.timeout));try{h=!1,c.send(a,l)}catch(e){if(h)throw e;l(-1,e)}}else l(-1,"No Transport");function l(e,t,n,r){var i,o,a,s,u,l=t;h||(h=!0,d&&C.clearTimeout(d),c=void 0,p=r||"",T.readyState=0<e?4:0,i=200<=e&&e<300||304===e,n&&(s=function(e,t,n){var r,i,o,a,s=e.contents,u=e.dataTypes;while("*"===u[0])u.shift(),void 0===r&&(r=e.mimeType||t.getResponseHeader("Content-Type"));if(r)for(i in s)if(s[i]&&s[i].test(r)){u.unshift(i);break}if(u[0]in n)o=u[0];else{for(i in n){if(!u[0]||e.converters[i+" "+u[0]]){o=i;break}a||(a=i)}o=o||a}if(o)return o!==u[0]&&u.unshift(o),n[o]}(v,T,n)),!i&&-1<S.inArray("script",v.dataTypes)&&S.inArray("json",v.dataTypes)<0&&(v.converters["text script"]=function(){}),s=function(e,t,n,r){var i,o,a,s,u,l={},c=e.dataTypes.slice();if(c[1])for(a in e.converters)l[a.toLowerCase()]=e.converters[a];o=c.shift();while(o)if(e.responseFields[o]&&(n[e.responseFields[o]]=t),!u&&r&&e.dataFilter&&(t=e.dataFilter(t,e.dataType)),u=o,o=c.shift())if("*"===o)o=u;else if("*"!==u&&u!==o){if(!(a=l[u+" "+o]||l["* "+o]))for(i in l)if((s=i.split(" "))[1]===o&&(a=l[u+" "+s[0]]||l["* "+s[0]])){!0===a?a=l[i]:!0!==l[i]&&(o=s[0],c.unshift(s[1]));break}if(!0!==a)if(a&&e["throws"])t=a(t);else try{t=a(t)}catch(e){return{state:"parsererror",error:a?e:"No conversion from "+u+" to "+o}}}return{state:"success",data:t}}(v,s,T,i),i?(v.ifModified&&((u=T.getResponseHeader("Last-Modified"))&&(S.lastModified[f]=u),(u=T.getResponseHeader("etag"))&&(S.etag[f]=u)),204===e||"HEAD"===v.type?l="nocontent":304===e?l="notmodified":(l=s.state,o=s.data,i=!(a=s.error))):(a=l,!e&&l||(l="error",e<0&&(e=0))),T.status=e,T.statusText=(t||l)+"",i?x.resolveWith(y,[o,l,T]):x.rejectWith(y,[T,l,a]),T.statusCode(w),w=void 0,g&&m.trigger(i?"ajaxSuccess":"ajaxError",[T,v,i?o:a]),b.fireWith(y,[T,l]),g&&(m.trigger("ajaxComplete",[T,v]),--S.active||S.event.trigger("ajaxStop")))}return T},getJSON:function(e,t,n){return S.get(e,t,n,"json")},getScript:function(e,t){return S.get(e,void 0,t,"script")}}),S.each(["get","post"],function(e,i){S[i]=function(e,t,n,r){return m(t)&&(r=r||n,n=t,t=void 0),S.ajax(S.extend({url:e,type:i,dataType:r,data:t,success:n},S.isPlainObject(e)&&e))}}),S.ajaxPrefilter(function(e){var t;for(t in e.headers)"content-type"===t.toLowerCase()&&(e.contentType=e.headers[t]||"")}),S._evalUrl=function(e,t,n){return S.ajax({url:e,type:"GET",dataType:"script",cache:!0,async:!1,global:!1,converters:{"text script":function(){}},dataFilter:function(e){S.globalEval(e,t,n)}})},S.fn.extend({wrapAll:function(e){var t;return this[0]&&(m(e)&&(e=e.call(this[0])),t=S(e,this[0].ownerDocument).eq(0).clone(!0),this[0].parentNode&&t.insertBefore(this[0]),t.map(function(){var e=this;while(e.firstElementChild)e=e.firstElementChild;return e}).append(this)),this},wrapInner:function(n){return m(n)?this.each(function(e){S(this).wrapInner(n.call(this,e))}):this.each(function(){var e=S(this),t=e.contents();t.length?t.wrapAll(n):e.append(n)})},wrap:function(t){var n=m(t);return this.each(function(e){S(this).wrapAll(n?t.call(this,e):t)})},unwrap:function(e){return this.parent(e).not("body").each(function(){S(this).replaceWith(this.childNodes)}),this}}),S.expr.pseudos.hidden=function(e){return!S.expr.pseudos.visible(e)},S.expr.pseudos.visible=function(e){return!!(e.offsetWidth||e.offsetHeight||e.getClientRects().length)},S.ajaxSettings.xhr=function(){try{return new C.XMLHttpRequest}catch(e){}};var Bt={0:200,1223:204},$t=S.ajaxSettings.xhr();y.cors=!!$t&&"withCredentials"in $t,y.ajax=$t=!!$t,S.ajaxTransport(function(i){var o,a;if(y.cors||$t&&!i.crossDomain)return{send:function(e,t){var n,r=i.xhr();if(r.open(i.type,i.url,i.async,i.username,i.password),i.xhrFields)for(n in i.xhrFields)r[n]=i.xhrFields[n];for(n in i.mimeType&&r.overrideMimeType&&r.overrideMimeType(i.mimeType),i.crossDomain||e["X-Requested-With"]||(e["X-Requested-With"]="XMLHttpRequest"),e)r.setRequestHeader(n,e[n]);o=function(e){return function(){o&&(o=a=r.onload=r.onerror=r.onabort=r.ontimeout=r.onreadystatechange=null,"abort"===e?r.abort():"error"===e?"number"!=typeof r.status?t(0,"error"):t(r.status,r.statusText):t(Bt[r.status]||r.status,r.statusText,"text"!==(r.responseType||"text")||"string"!=typeof r.responseText?{binary:r.response}:{text:r.responseText},r.getAllResponseHeaders()))}},r.onload=o(),a=r.onerror=r.ontimeout=o("error"),void 0!==r.onabort?r.onabort=a:r.onreadystatechange=function(){4===r.readyState&&C.setTimeout(function(){o&&a()})},o=o("abort");try{r.send(i.hasContent&&i.data||null)}catch(e){if(o)throw e}},abort:function(){o&&o()}}}),S.ajaxPrefilter(function(e){e.crossDomain&&(e.contents.script=!1)}),S.ajaxSetup({accepts:{script:"text/javascript, application/javascript, application/ecmascript, application/x-ecmascript"},contents:{script:/\b(?:java|ecma)script\b/},converters:{"text script":function(e){return S.globalEval(e),e}}}),S.ajaxPrefilter("script",function(e){void 0===e.cache&&(e.cache=!1),e.crossDomain&&(e.type="GET")}),S.ajaxTransport("script",function(n){var r,i;if(n.crossDomain||n.scriptAttrs)return{send:function(e,t){r=S("<script>").attr(n.scriptAttrs||{}).prop({charset:n.scriptCharset,src:n.url}).on("load error",i=function(e){r.remove(),i=null,e&&t("error"===e.type?404:200,e.type)}),E.head.appendChild(r[0])},abort:function(){i&&i()}}});var _t,zt=[],Ut=/(=)\?(?=&|$)|\?\?/;S.ajaxSetup({jsonp:"callback",jsonpCallback:function(){var e=zt.pop()||S.expando+"_"+wt.guid++;return this[e]=!0,e}}),S.ajaxPrefilter("json jsonp",function(e,t,n){var r,i,o,a=!1!==e.jsonp&&(Ut.test(e.url)?"url":"string"==typeof e.data&&0===(e.contentType||"").indexOf("application/x-www-form-urlencoded")&&Ut.test(e.data)&&"data");if(a||"jsonp"===e.dataTypes[0])return r=e.jsonpCallback=m(e.jsonpCallback)?e.jsonpCallback():e.jsonpCallback,a?e[a]=e[a].replace(Ut,"$1"+r):!1!==e.jsonp&&(e.url+=(Tt.test(e.url)?"&":"?")+e.jsonp+"="+r),e.converters["script json"]=function(){return o||S.error(r+" was not called"),o[0]},e.dataTypes[0]="json",i=C[r],C[r]=function(){o=arguments},n.always(function(){void 0===i?S(C).removeProp(r):C[r]=i,e[r]&&(e.jsonpCallback=t.jsonpCallback,zt.push(r)),o&&m(i)&&i(o[0]),o=i=void 0}),"script"}),y.createHTMLDocument=((_t=E.implementation.createHTMLDocument("").body).innerHTML="<form></form><form></form>",2===_t.childNodes.length),S.parseHTML=function(e,t,n){return"string"!=typeof e?[]:("boolean"==typeof t&&(n=t,t=!1),t||(y.createHTMLDocument?((r=(t=E.implementation.createHTMLDocument("")).createElement("base")).href=E.location.href,t.head.appendChild(r)):t=E),o=!n&&[],(i=N.exec(e))?[t.createElement(i[1])]:(i=xe([e],t,o),o&&o.length&&S(o).remove(),S.merge([],i.childNodes)));var r,i,o},S.fn.load=function(e,t,n){var r,i,o,a=this,s=e.indexOf(" ");return-1<s&&(r=ht(e.slice(s)),e=e.slice(0,s)),m(t)?(n=t,t=void 0):t&&"object"==typeof t&&(i="POST"),0<a.length&&S.ajax({url:e,type:i||"GET",dataType:"html",data:t}).done(function(e){o=arguments,a.html(r?S("<div>").append(S.parseHTML(e)).find(r):e)}).always(n&&function(e,t){a.each(function(){n.apply(this,o||[e.responseText,t,e])})}),this},S.expr.pseudos.animated=function(t){return S.grep(S.timers,function(e){return t===e.elem}).length},S.offset={setOffset:function(e,t,n){var r,i,o,a,s,u,l=S.css(e,"position"),c=S(e),f={};"static"===l&&(e.style.position="relative"),s=c.offset(),o=S.css(e,"top"),u=S.css(e,"left"),("absolute"===l||"fixed"===l)&&-1<(o+u).indexOf("auto")?(a=(r=c.position()).top,i=r.left):(a=parseFloat(o)||0,i=parseFloat(u)||0),m(t)&&(t=t.call(e,n,S.extend({},s))),null!=t.top&&(f.top=t.top-s.top+a),null!=t.left&&(f.left=t.left-s.left+i),"using"in t?t.using.call(e,f):c.css(f)}},S.fn.extend({offset:function(t){if(arguments.length)return void 0===t?this:this.each(function(e){S.offset.setOffset(this,t,e)});var e,n,r=this[0];return r?r.getClientRects().length?(e=r.getBoundingClientRect(),n=r.ownerDocument.defaultView,{top:e.top+n.pageYOffset,left:e.left+n.pageXOffset}):{top:0,left:0}:void 0},position:function(){if(this[0]){var e,t,n,r=this[0],i={top:0,left:0};if("fixed"===S.css(r,"position"))t=r.getBoundingClientRect();else{t=this.offset(),n=r.ownerDocument,e=r.offsetParent||n.documentElement;while(e&&(e===n.body||e===n.documentElement)&&"static"===S.css(e,"position"))e=e.parentNode;e&&e!==r&&1===e.nodeType&&((i=S(e).offset()).top+=S.css(e,"borderTopWidth",!0),i.left+=S.css(e,"borderLeftWidth",!0))}return{top:t.top-i.top-S.css(r,"marginTop",!0),left:t.left-i.left-S.css(r,"marginLeft",!0)}}},offsetParent:function(){return this.map(function(){var e=this.offsetParent;while(e&&"static"===S.css(e,"position"))e=e.offsetParent;return e||re})}}),S.each({scrollLeft:"pageXOffset",scrollTop:"pageYOffset"},function(t,i){var o="pageYOffset"===i;S.fn[t]=function(e){return $(this,function(e,t,n){var r;if(x(e)?r=e:9===e.nodeType&&(r=e.defaultView),void 0===n)return r?r[i]:e[t];r?r.scrollTo(o?r.pageXOffset:n,o?n:r.pageYOffset):e[t]=n},t,e,arguments.length)}}),S.each(["top","left"],function(e,n){S.cssHooks[n]=Fe(y.pixelPosition,function(e,t){if(t)return t=We(e,n),Pe.test(t)?S(e).position()[n]+"px":t})}),S.each({Height:"height",Width:"width"},function(a,s){S.each({padding:"inner"+a,content:s,"":"outer"+a},function(r,o){S.fn[o]=function(e,t){var n=arguments.length&&(r||"boolean"!=typeof e),i=r||(!0===e||!0===t?"margin":"border");return $(this,function(e,t,n){var r;return x(e)?0===o.indexOf("outer")?e["inner"+a]:e.document.documentElement["client"+a]:9===e.nodeType?(r=e.documentElement,Math.max(e.body["scroll"+a],r["scroll"+a],e.body["offset"+a],r["offset"+a],r["client"+a])):void 0===n?S.css(e,t,i):S.style(e,t,n,i)},s,n?e:void 0,n)}})}),S.each(["ajaxStart","ajaxStop","ajaxComplete","ajaxError","ajaxSuccess","ajaxSend"],function(e,t){S.fn[t]=function(e){return this.on(t,e)}}),S.fn.extend({bind:function(e,t,n){return this.on(e,null,t,n)},unbind:function(e,t){return this.off(e,null,t)},delegate:function(e,t,n,r){return this.on(t,e,n,r)},undelegate:function(e,t,n){return 1===arguments.length?this.off(e,"**"):this.off(t,e||"**",n)},hover:function(e,t){return this.mouseenter(e).mouseleave(t||e)}}),S.each("blur focus focusin focusout resize scroll click dblclick mousedown mouseup mousemove mouseover mouseout mouseenter mouseleave change select submit keydown keypress keyup contextmenu".split(" "),function(e,n){S.fn[n]=function(e,t){return 0<arguments.length?this.on(n,null,e,t):this.trigger(n)}});var Xt=/^[\s\uFEFF\xA0]+|[\s\uFEFF\xA0]+$/g;S.proxy=function(e,t){var n,r,i;if("string"==typeof t&&(n=e[t],t=e,e=n),m(e))return r=s.call(arguments,2),(i=function(){return e.apply(t||this,r.concat(s.call(arguments)))}).guid=e.guid=e.guid||S.guid++,i},S.holdReady=function(e){e?S.readyWait++:S.ready(!0)},S.isArray=Array.isArray,S.parseJSON=JSON.parse,S.nodeName=A,S.isFunction=m,S.isWindow=x,S.camelCase=X,S.type=w,S.now=Date.now,S.isNumeric=function(e){var t=S.type(e);return("number"===t||"string"===t)&&!isNaN(e-parseFloat(e))},S.trim=function(e){return null==e?"":(e+"").replace(Xt,"")},"function"==typeof define&&define.amd&&define("jquery",[],function(){return S});var Vt=C.jQuery,Gt=C.$;return S.noConflict=function(e){return C.$===S&&(C.$=Gt),e&&C.jQuery===S&&(C.jQuery=Vt),S},"undefined"==typeof e&&(C.jQuery=C.$=S),S});
diff --git a/license/index.html b/license/index.html
index d6f17f901..fa51a74ab 100644
--- a/license/index.html
+++ b/license/index.html
@@ -12,25 +12,22 @@
         <link href="../css/bootstrap.min.css" rel="stylesheet">
         <link href="../css/font-awesome.min.css" rel="stylesheet">
         <link href="../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../cinder/css/base.css" rel="stylesheet">
         <link href="../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../cinder/css/cinder.css" rel="stylesheet">
         <link href="../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../user_guide/evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../user_guide/evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../user_guide/evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../user_guide/evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../user_guide/math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../user_guide/math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -695,14 +692,16 @@ <h4 id="under-the-following-terms">Under the following terms:</h4>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../js/jquery-3.6.0.min.js"></script>
+        <script src="../js/bootstrap.min.js"></script>
         <script>
             var base_url = "..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../mathjaxhelper.js" defer></script>
-        <script src="../search/main.js" defer></script>
+        <script src="../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../mathjaxhelper.js"></script>
+        <script src="../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/search/search_index.json b/search/search_index.json
index 5d0b9c433..21c238775 100644
--- a/search/search_index.json
+++ b/search/search_index.json
@@ -1 +1 @@
-{"config":{"indexing":"full","lang":["en"],"min_search_length":3,"prebuild_index":false,"separator":"[\\s\\-]+"},"docs":[{"location":"","text":"Welcome to mlxtend's documentation! Mlxtend (machine learning extensions) is a Python library of useful tools for the day-to-day data science tasks. Links Documentation: https://rasbt.github.io/mlxtend Source code repository: https://github.com/rasbt/mlxtend PyPI: https://pypi.python.org/pypi/mlxtend Questions? Check out the GitHub Discussions board Examples import numpy as np import matplotlib.pyplot as plt import matplotlib.gridspec as gridspec import itertools from sklearn.linear_model import LogisticRegression from sklearn.svm import SVC from sklearn.ensemble import RandomForestClassifier from mlxtend.classifier import EnsembleVoteClassifier from mlxtend.data import iris_data from mlxtend.plotting import plot_decision_regions # Initializing Classifiers clf1 = LogisticRegression(random_state=0) clf2 = RandomForestClassifier(random_state=0) clf3 = SVC(random_state=0, probability=True) eclf = EnsembleVoteClassifier(clfs=[clf1, clf2, clf3], weights=[2, 1, 1], voting='soft') # Loading some example data X, y = iris_data() X = X[:,[0, 2]] # Plotting Decision Regions gs = gridspec.GridSpec(2, 2) fig = plt.figure(figsize=(10, 8)) labels = ['Logistic Regression', 'Random Forest', 'RBF kernel SVM', 'Ensemble'] for clf, lab, grd in zip([clf1, clf2, clf3, eclf], labels, itertools.product([0, 1], repeat=2)): clf.fit(X, y) ax = plt.subplot(gs[grd[0], grd[1]]) fig = plot_decision_regions(X=X, y=y, clf=clf, legend=2) plt.title(lab) plt.show() If you use mlxtend as part of your workflow in a scientific publication, please consider citing the mlxtend repository with the following DOI: @article{raschkas_2018_mlxtend, author = {Sebastian Raschka}, title = {MLxtend: Providing machine learning and data science utilities and extensions to Python\u2019s scientific computing stack}, journal = {The Journal of Open Source Software}, volume = {3}, number = {24}, month = apr, year = 2018, publisher = {The Open Journal}, doi = {10.21105/joss.00638}, url = {https://joss.theoj.org/papers/10.21105/joss.00638} } License This project is released under a permissive new BSD open source license ( LICENSE-BSD3.txt ) and commercially usable. There is no warranty; not even for merchantability or fitness for a particular purpose. In addition, you may use, copy, modify and redistribute all artistic creative works (figures and images) included in this distribution under the directory according to the terms and conditions of the Creative Commons Attribution 4.0 International License. See the file LICENSE-CC-BY.txt for details. (Computer-generated graphics such as the plots produced by matplotlib fall under the BSD license mentioned above). Contact I received a lot of feedback and questions about mlxtend recently, and I thought that it would be worthwhile to set up a public communication channel. Before you write an email with a question about mlxtend, please consider posting it here since it can also be useful to others! Please join the Google Groups Mailing List ! If Google Groups is not for you, please feel free to write me an email or consider filing an issue on GitHub's issue tracker for new feature requests or bug reports. In addition, I setup a Gitter channel for live discussions.","title":"Home"},{"location":"#welcome-to-mlxtends-documentation","text":"Mlxtend (machine learning extensions) is a Python library of useful tools for the day-to-day data science tasks.","title":"Welcome to mlxtend's documentation!"},{"location":"#links","text":"Documentation: https://rasbt.github.io/mlxtend Source code repository: https://github.com/rasbt/mlxtend PyPI: https://pypi.python.org/pypi/mlxtend Questions? Check out the GitHub Discussions board","title":"Links"},{"location":"#examples","text":"import numpy as np import matplotlib.pyplot as plt import matplotlib.gridspec as gridspec import itertools from sklearn.linear_model import LogisticRegression from sklearn.svm import SVC from sklearn.ensemble import RandomForestClassifier from mlxtend.classifier import EnsembleVoteClassifier from mlxtend.data import iris_data from mlxtend.plotting import plot_decision_regions # Initializing Classifiers clf1 = LogisticRegression(random_state=0) clf2 = RandomForestClassifier(random_state=0) clf3 = SVC(random_state=0, probability=True) eclf = EnsembleVoteClassifier(clfs=[clf1, clf2, clf3], weights=[2, 1, 1], voting='soft') # Loading some example data X, y = iris_data() X = X[:,[0, 2]] # Plotting Decision Regions gs = gridspec.GridSpec(2, 2) fig = plt.figure(figsize=(10, 8)) labels = ['Logistic Regression', 'Random Forest', 'RBF kernel SVM', 'Ensemble'] for clf, lab, grd in zip([clf1, clf2, clf3, eclf], labels, itertools.product([0, 1], repeat=2)): clf.fit(X, y) ax = plt.subplot(gs[grd[0], grd[1]]) fig = plot_decision_regions(X=X, y=y, clf=clf, legend=2) plt.title(lab) plt.show() If you use mlxtend as part of your workflow in a scientific publication, please consider citing the mlxtend repository with the following DOI: @article{raschkas_2018_mlxtend, author = {Sebastian Raschka}, title = {MLxtend: Providing machine learning and data science utilities and extensions to Python\u2019s scientific computing stack}, journal = {The Journal of Open Source Software}, volume = {3}, number = {24}, month = apr, year = 2018, publisher = {The Open Journal}, doi = {10.21105/joss.00638}, url = {https://joss.theoj.org/papers/10.21105/joss.00638} }","title":"Examples"},{"location":"#license","text":"This project is released under a permissive new BSD open source license ( LICENSE-BSD3.txt ) and commercially usable. There is no warranty; not even for merchantability or fitness for a particular purpose. In addition, you may use, copy, modify and redistribute all artistic creative works (figures and images) included in this distribution under the directory according to the terms and conditions of the Creative Commons Attribution 4.0 International License. See the file LICENSE-CC-BY.txt for details. (Computer-generated graphics such as the plots produced by matplotlib fall under the BSD license mentioned above).","title":"License"},{"location":"#contact","text":"I received a lot of feedback and questions about mlxtend recently, and I thought that it would be worthwhile to set up a public communication channel. Before you write an email with a question about mlxtend, please consider posting it here since it can also be useful to others! Please join the Google Groups Mailing List ! If Google Groups is not for you, please feel free to write me an email or consider filing an issue on GitHub's issue tracker for new feature requests or bug reports. In addition, I setup a Gitter channel for live discussions.","title":"Contact"},{"location":"CHANGELOG/","text":"Release Notes The CHANGELOG for the current development version is available at https://github.com/rasbt/mlxtend/blob/master/docs/sources/CHANGELOG.md . Version 0.23.2 (5 Nov 2024) Downloads Source code (zip) Source code (tar.gz) New Features and Enhancements Implement the FP-Growth and FP-Max algorithms with the possibility of missing values in the input dataset. Added a new metric Representativity for the association rules generated ( #1004 via zazass8 ). Files updated: ['mlxtend.frequent_patterns.fpcommon'] 'mlxtend.frequent_patterns.fpgrowth' 'mlxtend.frequent_patterns.fpmax' 'mlxtend/feature_selection/utilities.py' Modified _calc_score function to ensure compatibility with scikit-learn versions 1.4 and above by dynamically selecting between fit_params and params in cross_val_score . mlxtend.feature_selection.SequentialFeatureSelector Updated negative infinity constant to be compatible with old and new (>=2.0) numpy versions mlxtend.frequent_patterns.association_rules Implemented three new metrics: Jaccard, Certainty, and Kulczynski. ( #1096 ) Integrated scikit-learn's set_output method into TransactionEncoder ( #1087 via it176131 ) Changes [ mlxtend.frequent_patterns.fpcommon ] Added the null_values parameter in valid_input_check signature to check in case the input also includes null values. Changes the returns statements and function signatures for setup_fptree and generated_itemsets respectively to return the disabled array created and to include it as a parameter. Added code in [ mlxtend.frequent_patterns.fpcommon ] and mlxtend.frequent_patterns.association_rules to implement the algorithms in case null values exist when null_values is True. mlxtend.frequent_patterns.association_rules Added optional parameter 'return_metrics' to only return a given list of metrics, rather than every possible metric. Add n_classes_ attribute to stacking classifiers for compatibility with scikit-learn 1.3 ( #1091 ) Use Scipy's instead of NumPy's decompositions in PCA for improved accuracy in edge cases ( #1080 via [ fkdosilovic ]) Version 0.23.1 (5 Jan 2024) Downloads Source code (zip) Source code (tar.gz) Changes Updated dependency on distutils for python 3.12 and above ( #1072 via peanutsee ) Version 0.23.0 (22 Sep 2023) Downloads Source code (zip) Source code (tar.gz) Changes Address NumPy deprecations to make mlxtend compatible to NumPy 1.24 Changed the signature of the LinearRegression model of sklearn in the test removing the normalize parameter as it is deprecated. ( #1036 ) Add pyproject.toml to support PEP 518 builds ( #1065 via jmahlik ) Fixed installation from sdist failing ( #1065 via jmahlik ) Converted configuration to pyproject.toml ( #1065 via jmahlik ) Remove mlxtend.image submodule with face recognition functions due to poor dlib support in modern environments. New Features and Enhancements Document how to use SequentialFeatureSelector and multiclass ROC AUC. Version 0.22.0 (4 April 2023) Downloads Source code (zip) Source code (tar.gz) Changes When ExhaustiveFeatureSelector is run with n_jobs == 1 , joblib is now disabled, which enables more immediate (live) feedback when the verbose mode is enabled. ( #985 via Nima Sarajpoor ) Disabled unnecessary warning in EnsembleVoteClassifier ( #941 ) Fixed various documentation issues ( #849 and #951 via Lekshmanan Natarajan ) Fixed \"Edit on GitHub\" button ( #1024 ) New Features and Enhancements The mlxtend.frequent_patterns.association_rules function has a new metric - Zhang's Metric, which measures both association and dissociation. ( #980 ) Internal mlxtend.frequent_patterns.fpmax code improvement that avoids casting a sparse DataFrame into a dense NumPy array. ( #1000 via Tim Kellogg ) The plot_decision_regions function now has a n_jobs parameter to parallelize the computation. (In a particular use case, on a small dataset, there was a 21x speed-up (449 seconds vs 21 seconds on local HPC instance of 36 cores). ( #998 via Khalid ElHaj ) Added mlxtend.frequent_patterns.hmine algorithm and documentation for mining frequent itemsets using the H-Mine algorithm. ( #1020 via Fatih Sen ) Version 0.21.0 (09/17/2022) Downloads Source code (zip) Source code (tar.gz) New Features and Enhancements The mlxtend.evaluate.feature_importance_permutation function has a new feature_groups argument to treat user-specified feature groups as single features, which is useful for one-hot encoded features. ( #955 ) The mlxtend.feature_selection.ExhaustiveFeatureSelector and SequentialFeatureSelector also gained support for feature_groups with a behavior similar to the one described above. ( #957 and #965 via Nima Sarajpoor ) Changes The custom_feature_names parameter was removed from the ExhaustiveFeatureSelector due to redundancy and to simplify the code base. The ExhaustiveFeatureSelector documentation illustrates how the same behavior and outcome can be achieved using pandas DataFrames. ( #957 ) Bug Fixes None Version 0.20.0 (05/26/2022) Downloads Source code (zip) Source code (tar.gz) New Features and Enhancements The mlxtend.evaluate.bootstrap_point632_score now supports fit_params . ( #861 ) The mlxtend/plotting/decision_regions.py function now has a contourf_kwargs for matplotlib to change the look of the decision boundaries if desired. ( #881 via [ pbloem ]) Add a norm_colormap parameter to mlxtend.plotting.plot_confusion_matrix , to allow normalizing the colormap, e.g., using matplotlib.colors.LogNorm() ( #895 ) Add new GroupTimeSeriesSplit class for evaluation in time series tasks with support of custom groups and additional parameters in comparison with scikit-learn's TimeSeriesSplit . ( #915 via Dmitry Labazkin ) Changes Due to compatibility issues with newer package versions, certain functions from six.py have been removed so that mlxtend may not work anymore with Python 2.7. As an internal change to speed up unit testing, unit testing is now faciliated by GitHub workflows, and Travis CI and Appveyor hooks have been removed. Improved axis label rotation in mlxtend.plotting.heatmap and mlxtend.plotting.plot_confusion_matrix ( #872 ) Fix various typos in McNemar guides. Raises a warning if non-bool arrays are used in the frequent pattern functions apriori , fpmax , and fpgrowth . ( #934 via NimaSarajpoor ) Bug Fixes Fix unreadable labels in heatmap for certain colormaps. ( #852 ) Fix an issue in mlxtend.plotting.plot_confusion_matrix when string class names are passed ( #894 ) Version 0.19.0 (2021-09-02) Downloads Source code (zip) Source code (tar.gz) New Features Adds a second \"balanced accuracy\" interpretation (\"balanced\") to evaluate.accuracy_score in addition to the existing \"average\" option to compute the scikit-learn-style balanced accuracy. ( #764 ) Adds new scatter_hist function to mlxtend.plotting for generating a scattered histogram. ( #757 via Maitreyee Mhasaka ) The evaluate.permutation_test function now accepts a paired argument to specify to support paired permutation/randomization tests. ( #768 ) The StackingCVRegressor now also supports multi-dimensional targets similar to StackingRegressor via StackingCVRegressor(..., multi_output=True) . ( #802 via Marco Tiraboschi ) Changes Updates unit tests for scikit-learn 0.24.1 compatibility. ( #774 ) StackingRegressor now requires setting StackingRegressor(..., multi_output=True) if the target is multi-dimensional; this allows for better input validation. ( #802 ) Removes deprecated res argument from plot_decision_regions . ( #803 ) Adds a title_fontsize parameter to plot_learning_curves for controlling the title font size; also the plot style is now the matplotlib default. ( #818 ) Internal change using 'c': 'none' instead of 'c': '' in mlxtend.plotting.plot_decision_regions 's scatterplot highlights to stay compatible with Matplotlib 3.4 and newer. ( #822 ) Adds a fontcolor_threshold parameter to the mlxtend.plotting.plot_confusion_matrix function as an additional option for determining the font color cut-off manually. ( #827 ) The frequent_patterns.association_rules now raises a ValueError if an empty frequent itemset DataFrame is passed. ( #843 ) The .632 and .632+ bootstrap method implemented in the mlxtend.evaluate.bootstrap_point632_score function now use the whole training set for the resubstitution weighting term instead of the internal training set that is a new bootstrap sample in each round. ( #844 ) Bug Fixes Fixes a typo in the SequentialFeatureSelector documentation ( #835 via Jo\u00e3o Pedro Zanlorensi Cardoso ) Version 0.18.0 (2020-11-25) Downloads Source code (zip) Source code (tar.gz) New Features The bias_variance_decomp function now supports optional fit_params for the estimators that are fit on bootstrap samples. ( #748 ) The bias_variance_decomp function now supports Keras estimators. ( #725 via @hanzigs ) Adds new mlxtend.classifier.OneRClassifier (One Rule Classfier) class, a simple rule-based classifier that is often used as a performance baseline or simple interpretable model. ( #726 Adds new create_counterfactual method for creating counterfactuals to explain model predictions. ( #740 ) Changes permutation_test ( mlxtend.evaluate.permutation ) \u00ecs corrected to give the proportion of permutations whose statistic is at least as extreme as the one observed. ( #721 via Florian Charlier ) Fixes the McNemar confusion matrix layout to match the convention (and documentation), swapping the upper left and lower right cells. ( #744 via mmarius ) Bug Fixes The loss in LogisticRegression for logging purposes didn't include the L2 penalty for the first weight in the weight vector (this is not the bias unit). However, since this loss function was only used for logging purposes, and the gradient remains correct, this does not have an effect on the main code. ( #741 ) Fixes a bug in bias_variance_decomp where when the mse loss was used, downcasting to integers caused imprecise results for small numbers. ( #749 ) Version 0.17.3 (2020-07-27) Downloads Source code (zip) Source code (tar.gz) New Features Add predict_proba kwarg to bootstrap methods, to allow bootstrapping of scoring functions that take in probability values. ( #700 via Adam Li ) Add a cell_values parameter to mlxtend.plotting.heatmap() to optionally suppress cell annotations by setting cell_values=False . ( #703 Changes Implemented both use_clones and fit_base_estimators (previously refit in EnsembleVoteClassifier ) for EnsembleVoteClassifier and StackingClassifier . ( #670 via Katrina Ni ) Switched to using raw strings for regex in mlxtend.text to prevent deprecation warning in Python 3.8 ( #688 ) Slice data in sequential forward selection before sending to parallel backend, reducing memory consumption. Bug Fixes Fixes axis DeprecationWarning in matplotlib v3.1.0 and newer. ( #673 ) Fixes an issue with using meshgrid in no_information_rate function used by the bootstrap_point632_score function for the .632+ estimate. ( #688 ) Fixes an issue in fpmax that could lead to incorrect support values. ( #692 via Steve Harenberg ) Version 0.17.2 (2020-02-24) Downloads Source code (zip) Source code (tar.gz) New Features - Changes The previously deprecated OnehotTransactions has been removed in favor of the TransactionEncoder. Removed SparseDataFrame support in frequent pattern mining functions in favor of pandas >=1.0's new way for working sparse data. If you used SparseDataFrame formats, please see pandas' migration guide at https://pandas.pydata.org/pandas-docs/stable/user_guide/sparse.html#migrating. ( #667 ) The plot_confusion_matrix.py now also accepts a matplotlib figure and axis as input to which the confusion matrix plot can be added. ( #671 via Vahid Mirjalili ) Bug Fixes - Version 0.17.1 (2020-01-28) Downloads Source code (zip) Source code (tar.gz) New Features The SequentialFeatureSelector now supports using pre-specified feature sets via the fixed_features parameter. ( #578 ) Adds a new accuracy_score function to mlxtend.evaluate for computing basic classifcation accuracy, per-class accuracy, and average per-class accuracy. ( #624 via Deepan Das ) StackingClassifier and StackingCVClassifier now have a decision_function method, which serves as a preferred choice over predict_proba in calculating roc_auc and average_precision scores when the meta estimator is a linear model or support vector classifier. ( #634 via Qiang Gu ) Changes Improve the runtime performance for the apriori frequent itemset generating function when low_memory=True . Setting low_memory=False (default) is still faster for small itemsets, but low_memory=True can be much faster for large itemsets and requires less memory. Also, input validation for apriori , \u0300 fpgrowth and fpmax takes a significant amount of time when input pandas DataFrame is large; this is now dramatically reduced when input contains boolean values (and not zeros/ones), which is the case when using TransactionEncoder`. ( #619 via Denis Barbier ) Add support for newer sparse pandas DataFrame for frequent itemset algorithms. Also, input validation for apriori , \u0300 fpgrowth and fpmax` runs much faster on sparse DataFrame when input pandas DataFrame contains integer values. ( #621 via Denis Barbier ) Let fpgrowth and fpmax directly work on sparse DataFrame, they were previously converted into dense Numpy arrays. ( #622 via Denis Barbier ) Bug Fixes Fixes a bug in mlxtend.plotting.plot_pca_correlation_graph that caused the explaind variances not summing up to 1. Also, improves the runtime performance of the correlation computation and adds a missing function argument for the explained variances (eigenvalues) if users provide their own principal components. ( #593 via Gabriel Azevedo Ferreira ) Behavior of fpgrowth and apriori consistent for edgecases such as min_support=0 . ( #573 via Steve Harenberg ) fpmax returns an empty data frame now instead of raising an error if the frequent itemset set is empty. ( #573 via Steve Harenberg ) Fixes and issue in mlxtend.plotting.plot_confusion_matrix , where the font-color choice for medium-dark cells was not ideal and hard to read. #588 via sohrabtowfighi ) The svd mode of mlxtend.feature_extraction.PrincipalComponentAnalysis now also n-1 degrees of freedom instead of n d.o.f. when computing the eigenvalues to match the behavior of eigen . #595 Disable input validation for StackingCVClassifier because it causes issues if pipelines are used as input. #606 Version 0.17.0 (2019-07-19) Downloads Source code (zip) Source code (tar.gz) New Features Added an enhancement to the existing iris_data() such that both the UCI Repository version of the Iris dataset as well as the corrected, original version of the dataset can be loaded, which has a slight difference in two data points (consistent with Fisher's paper; this is also the same as in R). (via #539 via janismdhanbad ) Added optional groups parameter to SequentialFeatureSelector and ExhaustiveFeatureSelector fit() methods for forwarding to sklearn CV ( #537 via arc12 ) Added a new plot_pca_correlation_graph function to the mlxtend.plotting submodule for plotting a PCA correlation graph. ( #544 via Gabriel-Azevedo-Ferreira ) Added a zoom_factor parameter to the mlxten.plotting.plot_decision_region function that allows users to zoom in and out of the decision region plots. ( #545 ) Added a function fpgrowth that implements the FP-Growth algorithm for mining frequent itemsets as a drop-in replacement for the existing apriori algorithm. ( #550 via Steve Harenberg ) New heatmap function in mlxtend.plotting . ( #552 ) Added a function fpmax that implements the FP-Max algorithm for mining maximal itemsets as a drop-in replacement for the fpgrowth algorithm. ( #553 via Steve Harenberg ) New figsize parameter for the plot_decision_regions function in mlxtend.plotting . ( #555 via Mirza Hasanbasic ) New low_memory option for the apriori frequent itemset generating function. Setting low_memory=False (default) uses a substantially optimized version of the algorithm that is 3-6x faster than the original implementation ( low_memory=True ). ( #567 via jmayse ) Added numerically stable OLS methods which uses QR decomposition and Singular Value Decomposition (SVD) methods to LinearRegression in mlxtend.regressor.linear_regression . ( #575 via PuneetGrov3r ) Changes Now uses the latest joblib library under the hood for multiprocessing instead of sklearn.externals.joblib . ( #547 ) Changes to StackingCVClassifier and StackingCVRegressor such that first-level models are allowed to generate output of non-numeric type. ( #562 ) Bug Fixes Fixed documentation of iris_data() under iris.py by adding a note about differences in the iris data in R and UCI machine learning repo. Make sure that if the 'svd' mode is used in PCA, the number of eigenvalues is the same as when using 'eigen' (append 0's zeros in that case) ( #565 ) Version 0.16.0 (2019-05-12) Downloads Source code (zip) Source code (tar.gz) New Features StackingCVClassifier and StackingCVRegressor now support random_state parameter, which, together with shuffle , controls the randomness in the cv splitting. ( #523 via Qiang Gu ) StackingCVClassifier and StackingCVRegressor now have a new drop_last_proba parameter. It drops the last \"probability\" column in the feature set since if True , because it is redundant: p(y_c) = 1 - p(y_1) + p(y_2) + ... + p(y_{c-1}). This can be useful for meta-classifiers that are sensitive to perfectly collinear features. ( #532 ) Other stacking estimators, including StackingClassifier , StackingCVClassifier and StackingRegressor , support grid search over the regressors and even a single base regressor. ( #522 via Qiang Gu ) Adds multiprocessing support to StackingCVClassifier . ( #522 via Qiang Gu ) Adds multiprocessing support to StackingCVRegressor . ( #512 via Qiang Gu ) Now, the StackingCVRegressor also enables grid search over the regressors and even a single base regressor. When there are level-mixed parameters, GridSearchCV will try to replace hyperparameters in a top-down order (see the documentation for examples details). ( #515 via Qiang Gu ) Adds a verbose parameter to apriori to show the current iteration number as well as the itemset size currently being sampled. ( #519 Adds an optional class_name parameter to the confusion matrix function to display class names on the axis as tick marks. ( #487 via sandpiturtle ) Adds a pca.e_vals_normalized_ attribute to PCA for storing the eigenvalues also in normalized form; this is commonly referred to as variance explained ratios. #545 Changes Due to new features, restructuring, and better scikit-learn support (for GridSearchCV , etc.) the StackingCVRegressor 's meta regressor is now being accessed via 'meta_regressor__* in the parameter grid. E.g., if a RandomForestRegressor as meta- egressor was previously tuned via 'randomforestregressor__n_estimators' , this has now changed to 'meta_regressor__n_estimators' . ( #515 via Qiang Gu ) The same change mentioned above is now applied to other stacking estimators, including StackingClassifier , StackingCVClassifier and StackingRegressor . ( #522 via Qiang Gu ) Automatically performs mean centering for PCA solver 'SVD' such that using SVD is always equal to using the covariance matrix approach #545 Bug Fixes The feature_selection.ColumnSelector now also supports column names of type int (in addition to str names) if the input is a pandas DataFrame. ( #500 via tetrar124 Fix unreadable labels in plot_confusion_matrix for imbalanced datasets if show_absolute=True and show_normed=True . ( #504 ) Raises a more informative error if a SparseDataFrame is passed to apriori and the dataframe has integer column names that don't start with 0 due to current limitations of the SparseDataFrame implementation in pandas. ( #503 ) SequentialFeatureSelector now supports DataFrame as input for all operating modes (forward/backward/floating). #506 mlxtend.evaluate.feature_importance_permutation now correctly accepts scoring functions with proper function signature as metric argument. #528 Version 0.15.0 (2019-01-19) Downloads Source code (zip) Source code (tar.gz) New Features Adds a new transformer class to mlxtend.image , EyepadAlign , that aligns face images based on the location of the eyes. ( #466 by Vahid Mirjalili ) Adds a new function, mlxtend.evaluate.bias_variance_decomp that decomposes the loss of a regressor or classifier into bias and variance terms. ( #470 ) Adds a whitening parameter to PrincipalComponentAnalysis , to optionally whiten the transformed data such that the features have unit variance. ( #475 ) Changes Changed the default solver in PrincipalComponentAnalysis to 'svd' instead of 'eigen' to improve numerical stability. ( #474 ) The mlxtend.image.extract_face_landmarks now returns None if no facial landmarks were detected instead of an array of all zeros. ( #466 ) Bug Fixes The eigenvectors maybe have not been sorted in certain edge cases if solver was 'eigen' in PrincipalComponentAnalysis and LinearDiscriminantAnalysis . ( #477 , #478 ) Version 0.14.0 (2018-11-09) Downloads Source code (zip) Source code (tar.gz) New Features Added a scatterplotmatrix function to the plotting module. ( #437 ) Added sample_weight option to StackingRegressor , StackingClassifier , StackingCVRegressor , StackingCVClassifier , EnsembleVoteClassifier . ( #438 ) Added a RandomHoldoutSplit class to perform a random train/valid split without rotation in SequentialFeatureSelector , scikit-learn GridSearchCV etc. ( #442 ) Added a PredefinedHoldoutSplit class to perform a train/valid split, based on user-specified indices, without rotation in SequentialFeatureSelector , scikit-learn GridSearchCV etc. ( #443 ) Created a new mlxtend.image submodule for working on image processing-related tasks. ( #457 ) Added a new convenience function extract_face_landmarks based on dlib to mlxtend.image . ( #458 ) Added a method='oob' option to the mlxtend.evaluate.bootstrap_point632_score method to compute the classic out-of-bag bootstrap estimate ( #459 ) Added a method='.632+' option to the mlxtend.evaluate.bootstrap_point632_score method to compute the .632+ bootstrap estimate that addresses the optimism bias of the .632 bootstrap ( #459 ) Added a new mlxtend.evaluate.ftest function to perform an F-test for comparing the accuracies of two or more classification models. ( #460 ) Added a new mlxtend.evaluate.combined_ftest_5x2cv function to perform an combined 5x2cv F-Test for comparing the performance of two models. ( #461 ) Added a new mlxtend.evaluate.difference_proportions test for comparing two proportions (e.g., classifier accuracies) ( #462 ) Changes Addressed deprecations warnings in NumPy 0.15. ( #425 ) Because of complications in PR ( #459 ), Python 2.7 was now dropped; since official support for Python 2.7 by the Python Software Foundation is ending in approx. 12 months anyways, this re-focussing will hopefully free up some developer time with regard to not having to worry about backward compatibility Bug Fixes Fixed an issue with a missing import in mlxtend.plotting.plot_confusion_matrix . ( #428 ) Version 0.13.0 (2018-07-20) Downloads Source code (zip) Source code (tar.gz) New Features A meaningful error message is now raised when a cross-validation generator is used with SequentialFeatureSelector . ( #377 ) The SequentialFeatureSelector now accepts custom feature names via the fit method for more interpretable feature subset reports. ( #379 ) The SequentialFeatureSelector is now also compatible with Pandas DataFrames and uses DataFrame column-names for more interpretable feature subset reports. ( #379 ) ColumnSelector now works with Pandas DataFrames columns. ( #378 by Manuel Garrido ) The ExhaustiveFeatureSelector estimator in mlxtend.feature_selection now is safely stoppable mid-process by control+c. ( #380 ) Two new functions, vectorspace_orthonormalization and vectorspace_dimensionality were added to mlxtend.math to use the Gram-Schmidt process to convert a set of linearly independent vectors into a set of orthonormal basis vectors, and to compute the dimensionality of a vectorspace, respectively. ( #382 ) mlxtend.frequent_patterns.apriori now supports pandas SparseDataFrame s to generate frequent itemsets. ( #404 via Daniel Morales ) The plot_confusion_matrix function now has the ability to show normalized confusion matrix coefficients in addition to or instead of absolute confusion matrix coefficients with or without a colorbar. The text display method has been changed so that the full range of the colormap is used. The default size is also now set based on the number of classes. Added support for merging the meta features with the original input features in StackingRegressor (via use_features_in_secondary ) like it is already supported in the other Stacking classes. ( #418 ) Added a support_only to the association_rules function, which allow constructing association rules (based on the support metric only) for cropped input DataFrames that don't contain a complete set of antecedent and consequent support values. ( #421 ) Changes Itemsets generated with apriori are now frozenset s ( #393 by William Laney and #394 ) Now raises an error if a input DataFrame to apriori contains non 0, 1, True, False values. #419 ) Bug Fixes Allow mlxtend estimators to be cloned via scikit-learn's clone function. ( #374 ) Fixes bug to allow the correct use of refit=False in StackingRegressor and StackingCVRegressor ( #384 and ( #385 ) by selay01 ) Allow StackingClassifier to work with sparse matrices when use_features_in_secondary=True ( #408 by Floris Hoogenbook ) Allow StackingCVRegressor to work with sparse matrices when use_features_in_secondary=True ( #416 ) Allow StackingCVClassifier to work with sparse matrices when use_features_in_secondary=True ( #417 ) Version 0.12.0 (2018-21-04) Downloads Source code (zip) Source code (tar.gz) New Features A new feature_importance_permuation function to compute the feature importance in classifiers and regressors via the permutation importance method ( #358 ) The fit method of the ExhaustiveFeatureSelector now optionally accepts **fit_params for the estimator that is used for the feature selection. ( #354 by Zach Griffith) The fit method of the SequentialFeatureSelector now optionally accepts **fit_params for the estimator that is used for the feature selection. ( #350 by Zach Griffith) Changes Replaced plot_decision_regions colors by a colorblind-friendly palette and adds contour lines for decision regions. ( #348 ) All stacking estimators now raise NonFittedErrors if any method for inference is called prior to fitting the estimator. ( #353 ) Renamed the refit parameter of both the StackingClassifier and StackingCVClassifier to use_clones to be more explicit and less misleading. ( #368 ) Bug Fixes Various changes in the documentation and documentation tools to fix formatting issues ( #363 ) Fixes a bug where the StackingCVClassifier 's meta features were not stored in the original order when shuffle=True ( #370 ) Many documentation improvements, including links to the User Guides in the API docs ( #371 ) Version 0.11.0 (2018-03-14) Downloads Source code (zip) Source code (tar.gz) New Features New function implementing the resampled paired t-test procedure ( paired_ttest_resampled ) to compare the performance of two models. ( #323 ) New function implementing the k-fold paired t-test procedure ( paired_ttest_kfold_cv ) to compare the performance of two models (also called k-hold-out paired t-test). ( #324 ) New function implementing the 5x2cv paired t-test procedure ( paired_ttest_5x2cv ) proposed by Dieterrich (1998) to compare the performance of two models. ( #325 ) A refit parameter was added to stacking classes (similar to the refit parameter in the EnsembleVoteClassifier ), to support classifiers and regressors that follow the scikit-learn API but are not compatible with scikit-learn's clone function. ( #322 ) The ColumnSelector now has a drop_axis argument to use it in pipelines with CountVectorizers . ( #333 ) Changes Raises an informative error message if predict or predict_meta_features is called prior to calling the fit method in StackingRegressor and StackingCVRegressor . ( #315 ) The plot_decision_regions function now automatically determines the optimal setting based on the feature dimensions and supports anti-aliasing. The old res parameter has been deprecated. ( #309 by Guillaume Poirier-Morency ) Apriori code is faster due to optimization in onehot transformation and the amount of candidates generated by the apriori algorithm. ( #327 by Jakub Smid ) The OnehotTransactions class (which is typically often used in combination with the apriori function for association rule mining) is now more memory efficient as it uses boolean arrays instead of integer arrays. In addition, the OnehotTransactions class can be now be provided with sparse argument to generate sparse representations of the onehot matrix to further improve memory efficiency. ( #328 by Jakub Smid ) The OneHotTransactions has been deprecated and replaced by the TransactionEncoder . ( #332 The plot_decision_regions function now has three new parameters, scatter_kwargs , contourf_kwargs , and scatter_highlight_kwargs , that can be used to modify the plotting style. ( #342 by James Bourbeau ) Bug Fixes Fixed issue when class labels were provided to the EnsembleVoteClassifier when refit was set to false . ( #322 ) Allow arrays with 16-bit and 32-bit precision in plot_decision_regions function. ( #337 ) Fixed bug that raised an indexing error if the number of items was <= 1 when computing association rules using the conviction metric. ( #340 ) Version 0.10.0 (2017-12-22) Downloads Source code (zip) Source code (tar.gz) New Features New store_train_meta_features parameter for fit in StackingCVRegressor. if True, train meta-features are stored in self.train_meta_features_ . New pred_meta_features method for StackingCVRegressor . People can get test meta-features using this method. ( #294 via takashioya ) The new store_train_meta_features attribute and pred_meta_features method for the StackingCVRegressor were also added to the StackingRegressor , StackingClassifier , and StackingCVClassifier ( #299 & #300 ) New function ( evaluate.mcnemar_tables ) for creating multiple 2x2 contigency from model predictions arrays that can be used in multiple McNemar (post-hoc) tests or Cochran's Q or F tests, etc. ( #307 ) New function ( evaluate.cochrans_q ) for performing Cochran's Q test to compare the accuracy of multiple classifiers. ( #310 ) Changes Added requirements.txt to setup.py . ( #304 via Colin Carrol ) Bug Fixes Improved numerical stability for p-values computed via the the exact McNemar test ( #306 ) nose is not required to use the library ( #302 ) Version 0.9.1 (2017-11-19) Downloads Source code (zip) Source code (tar.gz) New Features Added mlxtend.evaluate.bootstrap_point632_score to evaluate the performance of estimators using the .632 bootstrap. ( #283 ) New max_len parameter for the frequent itemset generation via the apriori function to allow for early stopping. ( #270 ) Changes All feature index tuples in SequentialFeatureSelector or now in sorted order. ( #262 ) The SequentialFeatureSelector now runs the continuation of the floating inclusion/exclusion as described in Novovicova & Kittler (1994). Note that this didn't cause any difference in performance on any of the test scenarios but could lead to better performance in certain edge cases. ( #262 ) utils.Counter now accepts a name variable to help distinguish between multiple counters, time precision can be set with the 'precision' kwarg and the new attribute end_time holds the time the last iteration completed. ( #278 via Mathew Savage ) Bug Fixes Fixed an deprecation error that occured with McNemar test when using SciPy 1.0. ( #283 ) Version 0.9.0 (2017-10-21) Downloads Source code (zip) Source code (tar.gz) New Features Added evaluate.permutation_test , a permutation test for hypothesis testing (or A/B testing) to test if two samples come from the same distribution. Or in other words, a procedure to test the null hypothesis that that two groups are not significantly different (e.g., a treatment and a control group). ( #250 ) Added 'leverage' and 'conviction as evaluation metrics to the frequent_patterns.association_rules function. ( #246 & #247 ) Added a loadings_ attribute to PrincipalComponentAnalysis to compute the factor loadings of the features on the principal components. ( #251 ) Allow grid search over classifiers/regressors in ensemble and stacking estimators. ( #259 ) New make_multiplexer_dataset function that creates a dataset generated by a n-bit Boolean multiplexer for evaluating supervised learning algorithms. ( #263 ) Added a new BootstrapOutOfBag class, an implementation of the out-of-bag bootstrap to evaluate supervised learning algorithms. ( #265 ) The parameters for StackingClassifier , StackingCVClassifier , StackingRegressor , StackingCVRegressor , and EnsembleVoteClassifier can now be tuned using scikit-learn's GridSearchCV ( #254 via James Bourbeau ) Changes The 'support' column returned by frequent_patterns.association_rules was changed to compute the support of \"antecedant union consequent\", and new antecedant support' and 'consequent support' column were added to avoid ambiguity. ( #245 ) Allow the OnehotTransactions to be cloned via scikit-learn's clone function, which is required by e.g., scikit-learn's FeatureUnion or GridSearchCV (via Iaroslav Shcherbatyi ). ( #249 ) Bug Fixes Fix issues with self._init_time parameter in _IterativeModel subclasses. ( #256 ) Fix imprecision bug that occurred in plot_ecdf when run on Python 2.7. ( 264 ) The vectors from SVD in PrincipalComponentAnalysis are now being scaled so that the eigenvalues via solver='eigen' and solver='svd' now store eigenvalues that have the same magnitudes. ( #251 ) Version 0.8.0 (2017-09-09) Downloads Source code (zip) Source code (tar.gz) New Features Added a mlxtend.evaluate.bootstrap that implements the ordinary nonparametric bootstrap to bootstrap a single statistic (for example, the mean. median, R^2 of a regression fit, and so forth) #232 SequentialFeatureSelecor 's k_features now accepts a string argument \"best\" or \"parsimonious\" for more \"automated\" feature selection. For instance, if \"best\" is provided, the feature selector will return the feature subset with the best cross-validation performance. If \"parsimonious\" is provided as an argument, the smallest feature subset that is within one standard error of the cross-validation performance will be selected. #238 Changes SequentialFeatureSelector now uses np.nanmean over normal mean to support scorers that may return np.nan #211 (via mrkaiser ) The skip_if_stuck parameter was removed from SequentialFeatureSelector in favor of a more efficient implementation comparing the conditional inclusion/exclusion results (in the floating versions) to the performances of previously sampled feature sets that were cached #237 ExhaustiveFeatureSelector was modified to consume substantially less memory #195 (via Adam Erickson ) Bug Fixes Fixed a bug where the SequentialFeatureSelector selected a feature subset larger than then specified via the k_features tuple max-value #213 Version 0.7.0 (2017-06-22) Downloads Source code (zip) Source code (tar.gz) New Features New mlxtend.plotting.ecdf function for plotting empirical cumulative distribution functions ( #196 ). New StackingCVRegressor for stacking regressors with out-of-fold predictions to prevent overfitting ( #201 via Eike Dehling ). Changes The TensorFlow estimator have been removed from mlxtend, since TensorFlow has now very convenient ways to build on estimators, which render those implementations obsolete. plot_decision_regions now supports plotting decision regions for more than 2 training features #189 , via James Bourbeau ). Parallel execution in mlxtend.feature_selection.SequentialFeatureSelector and mlxtend.feature_selection.ExhaustiveFeatureSelector is now performed over different feature subsets instead of the different cross-validation folds to better utilize machines with multiple processors if the number of features is large ( #193 , via @whalebot-helmsman ). Raise meaningful error messages if pandas DataFrame s or Python lists of lists are fed into the StackingCVClassifer as a fit arguments ( 198 ). The n_folds parameter of the StackingCVClassifier was changed to cv and can now accept any kind of cross validation technique that is available from scikit-learn. For example, StackingCVClassifier(..., cv=StratifiedKFold(n_splits=3)) or StackingCVClassifier(..., cv=GroupKFold(n_splits=3)) ( #203 , via Konstantinos Paliouras ). Bug Fixes SequentialFeatureSelector now correctly accepts a None argument for the scoring parameter to infer the default scoring metric from scikit-learn classifiers and regressors ( #171 ). The plot_decision_regions function now supports pre-existing axes objects generated via matplotlib's plt.subplots . ( #184 , see example ) Made math.num_combinations and math.num_permutations numerically stable for large numbers of combinations and permutations ( #200 ). Version 0.6.0 (2017-03-18) Downloads Source code (zip) Source code (tar.gz) New Features An association_rules function is implemented that allows to generate rules based on a list of frequent itemsets (via Joshua Goerner ). Changes Adds a black edgecolor to plots via plotting.plot_decision_regions to make markers more distinguishable from the background in matplotlib>=2.0 . The association submodule was renamed to frequent_patterns . Bug Fixes The DataFrame index of apriori results are now unique and ordered. Fixed typos in autompg and wine datasets (via James Bourbeau ). Version 0.5.1 (2017-02-14) Downloads Source code (zip) Source code (tar.gz) New Features The EnsembleVoteClassifier has a new refit attribute that prevents refitting classifiers if refit=False to save computational time. Added a new lift_score function in evaluate to compute lift score (via Batuhan Bardak ). StackingClassifier and StackingRegressor support multivariate targets if the underlying models do (via kernc ). StackingClassifier has a new use_features_in_secondary attribute like StackingCVClassifier . Changes Changed default verbosity level in SequentialFeatureSelector to 0 The EnsembleVoteClassifier now raises a NotFittedError if the estimator wasn't fit before calling predict . (via Anton Loss ) Added new TensorFlow variable initialization syntax to guarantee compatibility with TensorFlow 1.0 Bug Fixes Fixed wrong default value for k_features in SequentialFeatureSelector Cast selected feature subsets in the SequentialFeautureSelector as sets to prevent the iterator from getting stuck if the k_idx are different permutations of the same combination (via Zac Wellmer ). Fixed an issue with learning curves that caused the performance metrics to be reversed (via ipashchenko ) Fixed a bug that could occur in the SequentialFeatureSelector if there are similarly-well performing subsets in the floating variants (via Zac Wellmer ). Version 0.5.0 (2016-11-09) Downloads Source code (zip) Source code (tar.gz) New Features New ExhaustiveFeatureSelector estimator in mlxtend.feature_selection for evaluating all feature combinations in a specified range The StackingClassifier has a new parameter average_probas that is set to True by default to maintain the current behavior. A deprecation warning was added though, and it will default to False in future releases (0.6.0); average_probas=False will result in stacking of the level-1 predicted probabilities rather than averaging these. New StackingCVClassifier estimator in 'mlxtend.classifier' for implementing a stacking ensemble that uses cross-validation techniques for training the meta-estimator to avoid overfitting ( Reiichiro Nakano ) New OnehotTransactions encoder class added to the preprocessing submodule for transforming transaction data into a one-hot encoded array The SequentialFeatureSelector estimator in mlxtend.feature_selection now is safely stoppable mid-process by control+c, and deprecated print_progress in favor of a more tunable verbose parameter ( Will McGinnis ) New apriori function in association to extract frequent itemsets from transaction data for association rule mining New checkerboard_plot function in plotting to plot checkerboard tables / heat maps New mcnemar_table and mcnemar functions in evaluate to compute 2x2 contingency tables and McNemar's test Changes All plotting functions have been moved to mlxtend.plotting for compatibility reasons with continuous integration services and to make the installation of matplotlib optional for users of mlxtend 's core functionality Added a compatibility layer for scikit-learn 0.18 using the new model_selection module while maintaining backwards compatibility to scikit-learn 0.17. Bug Fixes mlxtend.plotting.plot_decision_regions now draws decision regions correctly if more than 4 class labels are present Raise AttributeError in plot_decision_regions when the X_higlight argument is a 1D array ( chkoar ) Version 0.4.2 (2016-08-24) Downloads Source code (zip) Source code (tar.gz) PDF documentation New Features Added preprocessing.CopyTransformer , a mock class that returns copies of imput arrays via transform and fit_transform Changes Added AppVeyor to CI to ensure MS Windows compatibility Dataset are now saved as compressed .txt or .csv files rather than being imported as Python objects feature_selection.SequentialFeatureSelector now supports the selection of k_features using a tuple to specify a \"min-max\" k_features range Added \"SVD solver\" option to the PrincipalComponentAnalysis Raise a AttributeError with \"not fitted\" message in SequentialFeatureSelector if transform or get_metric_dict are called prior to fit Use small, positive bias units in TfMultiLayerPerceptron 's hidden layer(s) if the activations are ReLUs in order to avoid dead neurons Added an optional clone_estimator parameter to the SequentialFeatureSelector that defaults to True , avoiding the modification of the original estimator objects More rigorous type and shape checks in the evaluate.plot_decision_regions function DenseTransformer now doesn't raise and error if the input array is not sparse API clean-up using scikit-learn's BaseEstimator as parent class for feature_selection.ColumnSelector Bug Fixes Fixed a problem when a tuple-range was provided as argument to the SequentialFeatureSelector 's k_features parameter and the scoring metric was more negative than -1 (e.g., as in scikit-learn's MSE scoring function) (wahutch](https://github.com/wahutch)) Fixed an AttributeError issue when verbose > 1 in StackingClassifier Fixed a bug in classifier.SoftmaxRegression where the mean values of the offsets were used to update the bias units rather than their sum Fixed rare bug in MLP _layer_mapping functions that caused a swap between the random number generation seed when initializing weights and biases Version 0.4.1 (2016-05-01) Downloads Source code (zip) Source code (tar.gz) PDF documentation New Features New TensorFlow estimator for Linear Regression ( tf_regressor.TfLinearRegression ) New k-means clustering estimator ( cluster.Kmeans ) New TensorFlow k-means clustering estimator ( tf_cluster.Kmeans ) Changes Due to refactoring of the estimator classes, the init_weights parameter of the fit methods was globally renamed to init_params Overall performance improvements of estimators due to code clean-up and refactoring Added several additional checks for correct array types and more meaningful exception messages Added optional dropout to the tf_classifier.TfMultiLayerPerceptron classifier for regularization Added an optional decay parameter to the tf_classifier.TfMultiLayerPerceptron classifier for adaptive learning via an exponential decay of the learning rate eta Replaced old NeuralNetMLP by more streamlined MultiLayerPerceptron ( classifier.MultiLayerPerceptron ); now also with softmax in the output layer and categorical cross-entropy loss. Unified init_params parameter for fit functions to continue training where the algorithm left off (if supported) Version 0.4.0 (2016-04-09) New Features New TfSoftmaxRegression classifier using Tensorflow ( tf_classifier.TfSoftmaxRegression ) New SoftmaxRegression classifier ( classifier.SoftmaxRegression ) New TfMultiLayerPerceptron classifier using Tensorflow ( tf_classifier.TfMultiLayerPerceptron ) New StackingRegressor ( regressor.StackingRegressor ) New StackingClassifier ( classifier.StackingClassifier ) New function for one-hot encoding of class labels ( preprocessing.one_hot ) Added GridSearch support to the SequentialFeatureSelector ( feature_selection/.SequentialFeatureSelector ) evaluate.plot_decision_regions improvements: Function now handles class y-class labels correctly if array is of type float Correct handling of input arguments markers and colors Accept an existing Axes via the ax argument New print_progress parameter for all generalized models and multi-layer neural networks for printing time elapsed, ETA, and the current cost of the current epoch Minibatch learning for classifier.LogisticRegression , classifier.Adaline , and regressor.LinearRegression plus streamlined API New Principal Component Analysis class via mlxtend.feature_extraction.PrincipalComponentAnalysis New RBF Kernel Principal Component Analysis class via mlxtend.feature_extraction.RBFKernelPCA New Linear Discriminant Analysis class via mlxtend.feature_extraction.LinearDiscriminantAnalysis Changes The column parameter in mlxtend.preprocessing.standardize now defaults to None to standardize all columns more conveniently Version 0.3.0 (2016-01-31) Downloads Source code (zip) Source code (tar.gz) New Features Added a progress bar tracker to classifier.NeuralNetMLP Added a function to score predicted vs. target class labels evaluate.scoring Added confusion matrix functions to create ( evaluate.confusion_matrix ) and plot ( evaluate.plot_confusion_matrix ) confusion matrices New style parameter and improved axis scaling in mlxtend.evaluate.plot_learning_curves Added loadlocal_mnist to mlxtend.data for streaming MNIST from a local byte files into numpy arrays New NeuralNetMLP parameters: random_weights , shuffle_init , shuffle_epoch New SFS features such as the generation of pandas DataFrame results tables and plotting functions (with confidence intervals, standard deviation, and standard error bars) Added support for regression estimators in SFS Added Boston housing dataset New shuffle parameter for classifier.NeuralNetMLP Changes The mlxtend.preprocessing.standardize function now optionally returns the parameters, which are estimated from the array, for re-use. A further improvement makes the standardize function smarter in order to avoid zero-division errors Cosmetic improvements to the evaluate.plot_decision_regions function such as hiding plot axes Renaming of classifier.EnsembleClassfier to classifier.EnsembleVoteClassifier Improved random weight initialization in Perceptron , Adaline , LinearRegression , and LogisticRegression Changed learning parameter of mlxtend.classifier.Adaline to solver and added \"normal equation\" as closed-form solution solver Hide y-axis labels in mlxtend.evaluate.plot_decision_regions in 1 dimensional evaluations Sequential Feature Selection algorithms were unified into a single SequentialFeatureSelector class with parameters to enable floating selection and toggle between forward and backward selection. Stratified sampling of MNIST (now 500x random samples from each of the 10 digit categories) Renaming mlxtend.plotting to mlxtend.general_plotting in order to distinguish general plotting function from specialized utility function such as evaluate.plot_decision_regions Version 0.2.9 (2015-07-14) Downloads Source code (zip) Source code (tar.gz) New Features Sequential Feature Selection algorithms: SFS, SFFS, SBS, and SFBS Changes Changed regularization & lambda parameters in LogisticRegression to single parameter l2_lambda Version 0.2.8 (2015-06-27) API changes: mlxtend.sklearn.EnsembleClassifier -> mlxtend.classifier.EnsembleClassifier mlxtend.sklearn.ColumnSelector -> mlxtend.feature_selection.ColumnSelector mlxtend.sklearn.DenseTransformer -> mlxtend.preprocessing.DenseTransformer mlxtend.pandas.standardizing -> mlxtend.preprocessing.standardizing mlxtend.pandas.minmax_scaling -> mlxtend.preprocessing.minmax_scaling mlxtend.matplotlib -> mlxtend.plotting Added momentum learning parameter (alpha coefficient) to mlxtend.classifier.NeuralNetMLP . Added adaptive learning rate (decrease constant) to mlxtend.classifier.NeuralNetMLP . mlxtend.pandas.minmax_scaling became mlxtend.preprocessing.minmax_scaling and also supports NumPy arrays now mlxtend.pandas.standardizing became mlxtend.preprocessing.standardizing and now supports both NumPy arrays and pandas DataFrames; also, now ddof parameters to set the degrees of freedom when calculating the standard deviation Version 0.2.7 (2015-06-20) Added multilayer perceptron (feedforward artificial neural network) classifier as mlxtend.classifier.NeuralNetMLP . Added 5000 labeled trainingsamples from the MNIST handwritten digits dataset to mlxtend.data Version 0.2.6 (2015-05-08) Added ordinary least square regression using different solvers (gradient and stochastic gradient descent, and the closed form solution (normal equation) Added option for random weight initialization to logistic regression classifier and updated l2 regularization Added wine dataset to mlxtend.data Added invert_axes parameter mlxtend.matplotlib.enrichtment_plot to optionally plot the \"Count\" on the x-axis New verbose parameter for mlxtend.sklearn.EnsembleClassifier by Alejandro C. Bahnsen Added mlxtend.pandas.standardizing to standardize columns in a Pandas DataFrame Added parameters linestyles and markers to mlxtend.matplotlib.enrichment_plot mlxtend.regression.lin_regplot automatically adds np.newaxis and works w. python lists Added tokenizers: mlxtend.text.extract_emoticons and mlxtend.text.extract_words_and_emoticons Version 0.2.5 (2015-04-17) Added Sequential Backward Selection (mlxtend.sklearn.SBS) Added X_highlight parameter to mlxtend.evaluate.plot_decision_regions for highlighting test data points. Added mlxtend.regression.lin_regplot to plot the fitted line from linear regression. Added mlxtend.matplotlib.stacked_barplot to conveniently produce stacked barplots using pandas DataFrame s. Added mlxtend.matplotlib.enrichment_plot Version 0.2.4 (2015-03-15) Added scoring to mlxtend.evaluate.learning_curves (by user pfsq) Fixed setup.py bug caused by the missing README.html file matplotlib.category_scatter for pandas DataFrames and Numpy arrays Version 0.2.3 (2015-03-11) Added Logistic regression Gradient descent and stochastic gradient descent perceptron was changed to Adaline (Adaptive Linear Neuron) Perceptron and Adaline for {0, 1} classes Added mlxtend.preprocessing.shuffle_arrays_unison function to shuffle one or more NumPy arrays. Added shuffle and random seed parameter to stochastic gradient descent classifier. Added rstrip parameter to mlxtend.file_io.find_filegroups to allow trimming of base names. Added ignore_substring parameter to mlxtend.file_io.find_filegroups and find_files . Replaced .rstrip in mlxtend.file_io.find_filegroups with more robust regex. Gridsearch support for mlxtend.sklearn.EnsembleClassifier Version 0.2.2 (2015-03-01) Improved robustness of EnsembleClassifier. Extended plot_decision_regions() functionality for plotting 1D decision boundaries. Function matplotlib.plot_decision_regions was reorganized to evaluate.plot_decision_regions . evaluate.plot_learning_curves() function added. Added Rosenblatt, gradient descent, and stochastic gradient descent perceptrons. Version 0.2.1 (2015-01-20) Added mlxtend.pandas.minmax_scaling - a function to rescale pandas DataFrame columns. Slight update to the EnsembleClassifier interface (additional voting parameter) Fixed EnsembleClassifier to return correct class labels if class labels are not integers from 0 to n. Added new matplotlib function to plot decision regions of classifiers. Version 0.2.0 (2015-01-13) Improved mlxtend.text.generalize_duplcheck to remove duplicates and prevent endless looping issue. Added recursive search parameter to mlxtend.file_io.find_files. Added check_ext parameter mlxtend.file_io.find_files to search based on file extensions. Default parameter to ignore invisible files for mlxtend.file_io.find. Added transform and fit_transform to the EnsembleClassifier . Added mlxtend.file_io.find_filegroups function. Version 0.1.9 (2015-01-10) Implemented scikit-learn EnsembleClassifier (majority voting rule) class. Version 0.1.8 (2015-01-07) Improvements to mlxtend.text.generalize_names to handle certain Dutch last name prefixes (van, van der, de, etc.). Added mlxtend.text.generalize_name_duplcheck function to apply mlxtend.text.generalize_names function to a pandas DataFrame without creating duplicates. Version 0.1.7 (2015-01-07) Added text utilities with name generalization function. Added and file_io utilities. Version 0.1.6 (2015-01-04) Added combinations and permutations estimators. Version 0.1.5 (2014-12-11) Added DenseTransformer for pipelines and grid search. Version 0.1.4 (2014-08-20) mean_centering function is now a Class that creates MeanCenterer objects that can be used to fit data via the fit method, and center data at the column means via the transform and fit_transform method. Version 0.1.3 (2014-08-19) Added preprocessing module and mean_centering function. Version 0.1.2 (2014-08-19) Added matplotlib utilities and remove_borders function. Version 0.1.1 (2014-08-13) Simplified code for ColumnSelector.","title":"Release Notes"},{"location":"CHANGELOG/#release-notes","text":"The CHANGELOG for the current development version is available at https://github.com/rasbt/mlxtend/blob/master/docs/sources/CHANGELOG.md .","title":"Release Notes"},{"location":"CHANGELOG/#version-0232-5-nov-2024","text":"","title":"Version 0.23.2  (5 Nov 2024)"},{"location":"CHANGELOG/#downloads","text":"Source code (zip) Source code (tar.gz)","title":"Downloads"},{"location":"CHANGELOG/#new-features-and-enhancements","text":"Implement the FP-Growth and FP-Max algorithms with the possibility of missing values in the input dataset. Added a new metric Representativity for the association rules generated ( #1004 via zazass8 ). Files updated: ['mlxtend.frequent_patterns.fpcommon'] 'mlxtend.frequent_patterns.fpgrowth' 'mlxtend.frequent_patterns.fpmax' 'mlxtend/feature_selection/utilities.py' Modified _calc_score function to ensure compatibility with scikit-learn versions 1.4 and above by dynamically selecting between fit_params and params in cross_val_score . mlxtend.feature_selection.SequentialFeatureSelector Updated negative infinity constant to be compatible with old and new (>=2.0) numpy versions mlxtend.frequent_patterns.association_rules Implemented three new metrics: Jaccard, Certainty, and Kulczynski. ( #1096 ) Integrated scikit-learn's set_output method into TransactionEncoder ( #1087 via it176131 )","title":"New Features and Enhancements"},{"location":"CHANGELOG/#changes","text":"[ mlxtend.frequent_patterns.fpcommon ] Added the null_values parameter in valid_input_check signature to check in case the input also includes null values. Changes the returns statements and function signatures for setup_fptree and generated_itemsets respectively to return the disabled array created and to include it as a parameter. Added code in [ mlxtend.frequent_patterns.fpcommon ] and mlxtend.frequent_patterns.association_rules to implement the algorithms in case null values exist when null_values is True. mlxtend.frequent_patterns.association_rules Added optional parameter 'return_metrics' to only return a given list of metrics, rather than every possible metric. Add n_classes_ attribute to stacking classifiers for compatibility with scikit-learn 1.3 ( #1091 ) Use Scipy's instead of NumPy's decompositions in PCA for improved accuracy in edge cases ( #1080 via [ fkdosilovic ])","title":"Changes"},{"location":"CHANGELOG/#version-0231-5-jan-2024","text":"","title":"Version 0.23.1 (5 Jan 2024)"},{"location":"CHANGELOG/#downloads_1","text":"Source code (zip) Source code (tar.gz)","title":"Downloads"},{"location":"CHANGELOG/#changes_1","text":"Updated dependency on distutils for python 3.12 and above ( #1072 via peanutsee )","title":"Changes"},{"location":"CHANGELOG/#version-0230-22-sep-2023","text":"","title":"Version 0.23.0 (22 Sep 2023)"},{"location":"CHANGELOG/#downloads_2","text":"Source code (zip) Source code (tar.gz)","title":"Downloads"},{"location":"CHANGELOG/#changes_2","text":"Address NumPy deprecations to make mlxtend compatible to NumPy 1.24 Changed the signature of the LinearRegression model of sklearn in the test removing the normalize parameter as it is deprecated. ( #1036 ) Add pyproject.toml to support PEP 518 builds ( #1065 via jmahlik ) Fixed installation from sdist failing ( #1065 via jmahlik ) Converted configuration to pyproject.toml ( #1065 via jmahlik ) Remove mlxtend.image submodule with face recognition functions due to poor dlib support in modern environments.","title":"Changes"},{"location":"CHANGELOG/#new-features-and-enhancements_1","text":"Document how to use SequentialFeatureSelector and multiclass ROC AUC.","title":"New Features and Enhancements"},{"location":"CHANGELOG/#version-0220-4-april-2023","text":"","title":"Version 0.22.0 (4 April 2023)"},{"location":"CHANGELOG/#downloads_3","text":"Source code (zip) Source code (tar.gz)","title":"Downloads"},{"location":"CHANGELOG/#changes_3","text":"When ExhaustiveFeatureSelector is run with n_jobs == 1 , joblib is now disabled, which enables more immediate (live) feedback when the verbose mode is enabled. ( #985 via Nima Sarajpoor ) Disabled unnecessary warning in EnsembleVoteClassifier ( #941 ) Fixed various documentation issues ( #849 and #951 via Lekshmanan Natarajan ) Fixed \"Edit on GitHub\" button ( #1024 )","title":"Changes"},{"location":"CHANGELOG/#new-features-and-enhancements_2","text":"The mlxtend.frequent_patterns.association_rules function has a new metric - Zhang's Metric, which measures both association and dissociation. ( #980 ) Internal mlxtend.frequent_patterns.fpmax code improvement that avoids casting a sparse DataFrame into a dense NumPy array. ( #1000 via Tim Kellogg ) The plot_decision_regions function now has a n_jobs parameter to parallelize the computation. (In a particular use case, on a small dataset, there was a 21x speed-up (449 seconds vs 21 seconds on local HPC instance of 36 cores). ( #998 via Khalid ElHaj ) Added mlxtend.frequent_patterns.hmine algorithm and documentation for mining frequent itemsets using the H-Mine algorithm. ( #1020 via Fatih Sen )","title":"New Features and Enhancements"},{"location":"CHANGELOG/#version-0210-09172022","text":"","title":"Version 0.21.0 (09/17/2022)"},{"location":"CHANGELOG/#downloads_4","text":"Source code (zip) Source code (tar.gz)","title":"Downloads"},{"location":"CHANGELOG/#new-features-and-enhancements_3","text":"The mlxtend.evaluate.feature_importance_permutation function has a new feature_groups argument to treat user-specified feature groups as single features, which is useful for one-hot encoded features. ( #955 ) The mlxtend.feature_selection.ExhaustiveFeatureSelector and SequentialFeatureSelector also gained support for feature_groups with a behavior similar to the one described above. ( #957 and #965 via Nima Sarajpoor )","title":"New Features and Enhancements"},{"location":"CHANGELOG/#changes_4","text":"The custom_feature_names parameter was removed from the ExhaustiveFeatureSelector due to redundancy and to simplify the code base. The ExhaustiveFeatureSelector documentation illustrates how the same behavior and outcome can be achieved using pandas DataFrames. ( #957 )","title":"Changes"},{"location":"CHANGELOG/#bug-fixes","text":"None","title":"Bug Fixes"},{"location":"CHANGELOG/#version-0200-05262022","text":"","title":"Version 0.20.0 (05/26/2022)"},{"location":"CHANGELOG/#downloads_5","text":"Source code (zip) Source code (tar.gz)","title":"Downloads"},{"location":"CHANGELOG/#new-features-and-enhancements_4","text":"The mlxtend.evaluate.bootstrap_point632_score now supports fit_params . ( #861 ) The mlxtend/plotting/decision_regions.py function now has a contourf_kwargs for matplotlib to change the look of the decision boundaries if desired. ( #881 via [ pbloem ]) Add a norm_colormap parameter to mlxtend.plotting.plot_confusion_matrix , to allow normalizing the colormap, e.g., using matplotlib.colors.LogNorm() ( #895 ) Add new GroupTimeSeriesSplit class for evaluation in time series tasks with support of custom groups and additional parameters in comparison with scikit-learn's TimeSeriesSplit . ( #915 via Dmitry Labazkin )","title":"New Features and Enhancements"},{"location":"CHANGELOG/#changes_5","text":"Due to compatibility issues with newer package versions, certain functions from six.py have been removed so that mlxtend may not work anymore with Python 2.7. As an internal change to speed up unit testing, unit testing is now faciliated by GitHub workflows, and Travis CI and Appveyor hooks have been removed. Improved axis label rotation in mlxtend.plotting.heatmap and mlxtend.plotting.plot_confusion_matrix ( #872 ) Fix various typos in McNemar guides. Raises a warning if non-bool arrays are used in the frequent pattern functions apriori , fpmax , and fpgrowth . ( #934 via NimaSarajpoor )","title":"Changes"},{"location":"CHANGELOG/#bug-fixes_1","text":"Fix unreadable labels in heatmap for certain colormaps. ( #852 ) Fix an issue in mlxtend.plotting.plot_confusion_matrix when string class names are passed ( #894 )","title":"Bug Fixes"},{"location":"CHANGELOG/#version-0190-2021-09-02","text":"","title":"Version 0.19.0 (2021-09-02)"},{"location":"CHANGELOG/#downloads_6","text":"Source code (zip) Source code (tar.gz)","title":"Downloads"},{"location":"CHANGELOG/#new-features","text":"Adds a second \"balanced accuracy\" interpretation (\"balanced\") to evaluate.accuracy_score in addition to the existing \"average\" option to compute the scikit-learn-style balanced accuracy. ( #764 ) Adds new scatter_hist function to mlxtend.plotting for generating a scattered histogram. ( #757 via Maitreyee Mhasaka ) The evaluate.permutation_test function now accepts a paired argument to specify to support paired permutation/randomization tests. ( #768 ) The StackingCVRegressor now also supports multi-dimensional targets similar to StackingRegressor via StackingCVRegressor(..., multi_output=True) . ( #802 via Marco Tiraboschi )","title":"New Features"},{"location":"CHANGELOG/#changes_6","text":"Updates unit tests for scikit-learn 0.24.1 compatibility. ( #774 ) StackingRegressor now requires setting StackingRegressor(..., multi_output=True) if the target is multi-dimensional; this allows for better input validation. ( #802 ) Removes deprecated res argument from plot_decision_regions . ( #803 ) Adds a title_fontsize parameter to plot_learning_curves for controlling the title font size; also the plot style is now the matplotlib default. ( #818 ) Internal change using 'c': 'none' instead of 'c': '' in mlxtend.plotting.plot_decision_regions 's scatterplot highlights to stay compatible with Matplotlib 3.4 and newer. ( #822 ) Adds a fontcolor_threshold parameter to the mlxtend.plotting.plot_confusion_matrix function as an additional option for determining the font color cut-off manually. ( #827 ) The frequent_patterns.association_rules now raises a ValueError if an empty frequent itemset DataFrame is passed. ( #843 ) The .632 and .632+ bootstrap method implemented in the mlxtend.evaluate.bootstrap_point632_score function now use the whole training set for the resubstitution weighting term instead of the internal training set that is a new bootstrap sample in each round. ( #844 )","title":"Changes"},{"location":"CHANGELOG/#bug-fixes_2","text":"Fixes a typo in the SequentialFeatureSelector documentation ( #835 via Jo\u00e3o Pedro Zanlorensi Cardoso )","title":"Bug Fixes"},{"location":"CHANGELOG/#version-0180-2020-11-25","text":"","title":"Version 0.18.0 (2020-11-25)"},{"location":"CHANGELOG/#downloads_7","text":"Source code (zip) Source code (tar.gz)","title":"Downloads"},{"location":"CHANGELOG/#new-features_1","text":"The bias_variance_decomp function now supports optional fit_params for the estimators that are fit on bootstrap samples. ( #748 ) The bias_variance_decomp function now supports Keras estimators. ( #725 via @hanzigs ) Adds new mlxtend.classifier.OneRClassifier (One Rule Classfier) class, a simple rule-based classifier that is often used as a performance baseline or simple interpretable model. ( #726 Adds new create_counterfactual method for creating counterfactuals to explain model predictions. ( #740 )","title":"New Features"},{"location":"CHANGELOG/#changes_7","text":"permutation_test ( mlxtend.evaluate.permutation ) \u00ecs corrected to give the proportion of permutations whose statistic is at least as extreme as the one observed. ( #721 via Florian Charlier ) Fixes the McNemar confusion matrix layout to match the convention (and documentation), swapping the upper left and lower right cells. ( #744 via mmarius )","title":"Changes"},{"location":"CHANGELOG/#bug-fixes_3","text":"The loss in LogisticRegression for logging purposes didn't include the L2 penalty for the first weight in the weight vector (this is not the bias unit). However, since this loss function was only used for logging purposes, and the gradient remains correct, this does not have an effect on the main code. ( #741 ) Fixes a bug in bias_variance_decomp where when the mse loss was used, downcasting to integers caused imprecise results for small numbers. ( #749 )","title":"Bug Fixes"},{"location":"CHANGELOG/#version-0173-2020-07-27","text":"","title":"Version 0.17.3 (2020-07-27)"},{"location":"CHANGELOG/#downloads_8","text":"Source code (zip) Source code (tar.gz)","title":"Downloads"},{"location":"CHANGELOG/#new-features_2","text":"Add predict_proba kwarg to bootstrap methods, to allow bootstrapping of scoring functions that take in probability values. ( #700 via Adam Li ) Add a cell_values parameter to mlxtend.plotting.heatmap() to optionally suppress cell annotations by setting cell_values=False . ( #703","title":"New Features"},{"location":"CHANGELOG/#changes_8","text":"Implemented both use_clones and fit_base_estimators (previously refit in EnsembleVoteClassifier ) for EnsembleVoteClassifier and StackingClassifier . ( #670 via Katrina Ni ) Switched to using raw strings for regex in mlxtend.text to prevent deprecation warning in Python 3.8 ( #688 ) Slice data in sequential forward selection before sending to parallel backend, reducing memory consumption.","title":"Changes"},{"location":"CHANGELOG/#bug-fixes_4","text":"Fixes axis DeprecationWarning in matplotlib v3.1.0 and newer. ( #673 ) Fixes an issue with using meshgrid in no_information_rate function used by the bootstrap_point632_score function for the .632+ estimate. ( #688 ) Fixes an issue in fpmax that could lead to incorrect support values. ( #692 via Steve Harenberg )","title":"Bug Fixes"},{"location":"CHANGELOG/#version-0172-2020-02-24","text":"","title":"Version 0.17.2 (2020-02-24)"},{"location":"CHANGELOG/#downloads_9","text":"Source code (zip) Source code (tar.gz)","title":"Downloads"},{"location":"CHANGELOG/#new-features_3","text":"-","title":"New Features"},{"location":"CHANGELOG/#changes_9","text":"The previously deprecated OnehotTransactions has been removed in favor of the TransactionEncoder. Removed SparseDataFrame support in frequent pattern mining functions in favor of pandas >=1.0's new way for working sparse data. If you used SparseDataFrame formats, please see pandas' migration guide at https://pandas.pydata.org/pandas-docs/stable/user_guide/sparse.html#migrating. ( #667 ) The plot_confusion_matrix.py now also accepts a matplotlib figure and axis as input to which the confusion matrix plot can be added. ( #671 via Vahid Mirjalili )","title":"Changes"},{"location":"CHANGELOG/#bug-fixes_5","text":"-","title":"Bug Fixes"},{"location":"CHANGELOG/#version-0171-2020-01-28","text":"","title":"Version 0.17.1 (2020-01-28)"},{"location":"CHANGELOG/#downloads_10","text":"Source code (zip) Source code (tar.gz)","title":"Downloads"},{"location":"CHANGELOG/#new-features_4","text":"The SequentialFeatureSelector now supports using pre-specified feature sets via the fixed_features parameter. ( #578 ) Adds a new accuracy_score function to mlxtend.evaluate for computing basic classifcation accuracy, per-class accuracy, and average per-class accuracy. ( #624 via Deepan Das ) StackingClassifier and StackingCVClassifier now have a decision_function method, which serves as a preferred choice over predict_proba in calculating roc_auc and average_precision scores when the meta estimator is a linear model or support vector classifier. ( #634 via Qiang Gu )","title":"New Features"},{"location":"CHANGELOG/#changes_10","text":"Improve the runtime performance for the apriori frequent itemset generating function when low_memory=True . Setting low_memory=False (default) is still faster for small itemsets, but low_memory=True can be much faster for large itemsets and requires less memory. Also, input validation for apriori , \u0300 fpgrowth and fpmax takes a significant amount of time when input pandas DataFrame is large; this is now dramatically reduced when input contains boolean values (and not zeros/ones), which is the case when using TransactionEncoder`. ( #619 via Denis Barbier ) Add support for newer sparse pandas DataFrame for frequent itemset algorithms. Also, input validation for apriori , \u0300 fpgrowth and fpmax` runs much faster on sparse DataFrame when input pandas DataFrame contains integer values. ( #621 via Denis Barbier ) Let fpgrowth and fpmax directly work on sparse DataFrame, they were previously converted into dense Numpy arrays. ( #622 via Denis Barbier )","title":"Changes"},{"location":"CHANGELOG/#bug-fixes_6","text":"Fixes a bug in mlxtend.plotting.plot_pca_correlation_graph that caused the explaind variances not summing up to 1. Also, improves the runtime performance of the correlation computation and adds a missing function argument for the explained variances (eigenvalues) if users provide their own principal components. ( #593 via Gabriel Azevedo Ferreira ) Behavior of fpgrowth and apriori consistent for edgecases such as min_support=0 . ( #573 via Steve Harenberg ) fpmax returns an empty data frame now instead of raising an error if the frequent itemset set is empty. ( #573 via Steve Harenberg ) Fixes and issue in mlxtend.plotting.plot_confusion_matrix , where the font-color choice for medium-dark cells was not ideal and hard to read. #588 via sohrabtowfighi ) The svd mode of mlxtend.feature_extraction.PrincipalComponentAnalysis now also n-1 degrees of freedom instead of n d.o.f. when computing the eigenvalues to match the behavior of eigen . #595 Disable input validation for StackingCVClassifier because it causes issues if pipelines are used as input. #606","title":"Bug Fixes"},{"location":"CHANGELOG/#version-0170-2019-07-19","text":"","title":"Version 0.17.0 (2019-07-19)"},{"location":"CHANGELOG/#downloads_11","text":"Source code (zip) Source code (tar.gz)","title":"Downloads"},{"location":"CHANGELOG/#new-features_5","text":"Added an enhancement to the existing iris_data() such that both the UCI Repository version of the Iris dataset as well as the corrected, original version of the dataset can be loaded, which has a slight difference in two data points (consistent with Fisher's paper; this is also the same as in R). (via #539 via janismdhanbad ) Added optional groups parameter to SequentialFeatureSelector and ExhaustiveFeatureSelector fit() methods for forwarding to sklearn CV ( #537 via arc12 ) Added a new plot_pca_correlation_graph function to the mlxtend.plotting submodule for plotting a PCA correlation graph. ( #544 via Gabriel-Azevedo-Ferreira ) Added a zoom_factor parameter to the mlxten.plotting.plot_decision_region function that allows users to zoom in and out of the decision region plots. ( #545 ) Added a function fpgrowth that implements the FP-Growth algorithm for mining frequent itemsets as a drop-in replacement for the existing apriori algorithm. ( #550 via Steve Harenberg ) New heatmap function in mlxtend.plotting . ( #552 ) Added a function fpmax that implements the FP-Max algorithm for mining maximal itemsets as a drop-in replacement for the fpgrowth algorithm. ( #553 via Steve Harenberg ) New figsize parameter for the plot_decision_regions function in mlxtend.plotting . ( #555 via Mirza Hasanbasic ) New low_memory option for the apriori frequent itemset generating function. Setting low_memory=False (default) uses a substantially optimized version of the algorithm that is 3-6x faster than the original implementation ( low_memory=True ). ( #567 via jmayse ) Added numerically stable OLS methods which uses QR decomposition and Singular Value Decomposition (SVD) methods to LinearRegression in mlxtend.regressor.linear_regression . ( #575 via PuneetGrov3r )","title":"New Features"},{"location":"CHANGELOG/#changes_11","text":"Now uses the latest joblib library under the hood for multiprocessing instead of sklearn.externals.joblib . ( #547 ) Changes to StackingCVClassifier and StackingCVRegressor such that first-level models are allowed to generate output of non-numeric type. ( #562 )","title":"Changes"},{"location":"CHANGELOG/#bug-fixes_7","text":"Fixed documentation of iris_data() under iris.py by adding a note about differences in the iris data in R and UCI machine learning repo. Make sure that if the 'svd' mode is used in PCA, the number of eigenvalues is the same as when using 'eigen' (append 0's zeros in that case) ( #565 )","title":"Bug Fixes"},{"location":"CHANGELOG/#version-0160-2019-05-12","text":"","title":"Version 0.16.0 (2019-05-12)"},{"location":"CHANGELOG/#downloads_12","text":"Source code (zip) Source code (tar.gz)","title":"Downloads"},{"location":"CHANGELOG/#new-features_6","text":"StackingCVClassifier and StackingCVRegressor now support random_state parameter, which, together with shuffle , controls the randomness in the cv splitting. ( #523 via Qiang Gu ) StackingCVClassifier and StackingCVRegressor now have a new drop_last_proba parameter. It drops the last \"probability\" column in the feature set since if True , because it is redundant: p(y_c) = 1 - p(y_1) + p(y_2) + ... + p(y_{c-1}). This can be useful for meta-classifiers that are sensitive to perfectly collinear features. ( #532 ) Other stacking estimators, including StackingClassifier , StackingCVClassifier and StackingRegressor , support grid search over the regressors and even a single base regressor. ( #522 via Qiang Gu ) Adds multiprocessing support to StackingCVClassifier . ( #522 via Qiang Gu ) Adds multiprocessing support to StackingCVRegressor . ( #512 via Qiang Gu ) Now, the StackingCVRegressor also enables grid search over the regressors and even a single base regressor. When there are level-mixed parameters, GridSearchCV will try to replace hyperparameters in a top-down order (see the documentation for examples details). ( #515 via Qiang Gu ) Adds a verbose parameter to apriori to show the current iteration number as well as the itemset size currently being sampled. ( #519 Adds an optional class_name parameter to the confusion matrix function to display class names on the axis as tick marks. ( #487 via sandpiturtle ) Adds a pca.e_vals_normalized_ attribute to PCA for storing the eigenvalues also in normalized form; this is commonly referred to as variance explained ratios. #545","title":"New Features"},{"location":"CHANGELOG/#changes_12","text":"Due to new features, restructuring, and better scikit-learn support (for GridSearchCV , etc.) the StackingCVRegressor 's meta regressor is now being accessed via 'meta_regressor__* in the parameter grid. E.g., if a RandomForestRegressor as meta- egressor was previously tuned via 'randomforestregressor__n_estimators' , this has now changed to 'meta_regressor__n_estimators' . ( #515 via Qiang Gu ) The same change mentioned above is now applied to other stacking estimators, including StackingClassifier , StackingCVClassifier and StackingRegressor . ( #522 via Qiang Gu ) Automatically performs mean centering for PCA solver 'SVD' such that using SVD is always equal to using the covariance matrix approach #545","title":"Changes"},{"location":"CHANGELOG/#bug-fixes_8","text":"The feature_selection.ColumnSelector now also supports column names of type int (in addition to str names) if the input is a pandas DataFrame. ( #500 via tetrar124 Fix unreadable labels in plot_confusion_matrix for imbalanced datasets if show_absolute=True and show_normed=True . ( #504 ) Raises a more informative error if a SparseDataFrame is passed to apriori and the dataframe has integer column names that don't start with 0 due to current limitations of the SparseDataFrame implementation in pandas. ( #503 ) SequentialFeatureSelector now supports DataFrame as input for all operating modes (forward/backward/floating). #506 mlxtend.evaluate.feature_importance_permutation now correctly accepts scoring functions with proper function signature as metric argument. #528","title":"Bug Fixes"},{"location":"CHANGELOG/#version-0150-2019-01-19","text":"","title":"Version 0.15.0 (2019-01-19)"},{"location":"CHANGELOG/#downloads_13","text":"Source code (zip) Source code (tar.gz)","title":"Downloads"},{"location":"CHANGELOG/#new-features_7","text":"Adds a new transformer class to mlxtend.image , EyepadAlign , that aligns face images based on the location of the eyes. ( #466 by Vahid Mirjalili ) Adds a new function, mlxtend.evaluate.bias_variance_decomp that decomposes the loss of a regressor or classifier into bias and variance terms. ( #470 ) Adds a whitening parameter to PrincipalComponentAnalysis , to optionally whiten the transformed data such that the features have unit variance. ( #475 )","title":"New Features"},{"location":"CHANGELOG/#changes_13","text":"Changed the default solver in PrincipalComponentAnalysis to 'svd' instead of 'eigen' to improve numerical stability. ( #474 ) The mlxtend.image.extract_face_landmarks now returns None if no facial landmarks were detected instead of an array of all zeros. ( #466 )","title":"Changes"},{"location":"CHANGELOG/#bug-fixes_9","text":"The eigenvectors maybe have not been sorted in certain edge cases if solver was 'eigen' in PrincipalComponentAnalysis and LinearDiscriminantAnalysis . ( #477 , #478 )","title":"Bug Fixes"},{"location":"CHANGELOG/#version-0140-2018-11-09","text":"","title":"Version 0.14.0 (2018-11-09)"},{"location":"CHANGELOG/#downloads_14","text":"Source code (zip) Source code (tar.gz)","title":"Downloads"},{"location":"CHANGELOG/#new-features_8","text":"Added a scatterplotmatrix function to the plotting module. ( #437 ) Added sample_weight option to StackingRegressor , StackingClassifier , StackingCVRegressor , StackingCVClassifier , EnsembleVoteClassifier . ( #438 ) Added a RandomHoldoutSplit class to perform a random train/valid split without rotation in SequentialFeatureSelector , scikit-learn GridSearchCV etc. ( #442 ) Added a PredefinedHoldoutSplit class to perform a train/valid split, based on user-specified indices, without rotation in SequentialFeatureSelector , scikit-learn GridSearchCV etc. ( #443 ) Created a new mlxtend.image submodule for working on image processing-related tasks. ( #457 ) Added a new convenience function extract_face_landmarks based on dlib to mlxtend.image . ( #458 ) Added a method='oob' option to the mlxtend.evaluate.bootstrap_point632_score method to compute the classic out-of-bag bootstrap estimate ( #459 ) Added a method='.632+' option to the mlxtend.evaluate.bootstrap_point632_score method to compute the .632+ bootstrap estimate that addresses the optimism bias of the .632 bootstrap ( #459 ) Added a new mlxtend.evaluate.ftest function to perform an F-test for comparing the accuracies of two or more classification models. ( #460 ) Added a new mlxtend.evaluate.combined_ftest_5x2cv function to perform an combined 5x2cv F-Test for comparing the performance of two models. ( #461 ) Added a new mlxtend.evaluate.difference_proportions test for comparing two proportions (e.g., classifier accuracies) ( #462 )","title":"New Features"},{"location":"CHANGELOG/#changes_14","text":"Addressed deprecations warnings in NumPy 0.15. ( #425 ) Because of complications in PR ( #459 ), Python 2.7 was now dropped; since official support for Python 2.7 by the Python Software Foundation is ending in approx. 12 months anyways, this re-focussing will hopefully free up some developer time with regard to not having to worry about backward compatibility","title":"Changes"},{"location":"CHANGELOG/#bug-fixes_10","text":"Fixed an issue with a missing import in mlxtend.plotting.plot_confusion_matrix . ( #428 )","title":"Bug Fixes"},{"location":"CHANGELOG/#version-0130-2018-07-20","text":"","title":"Version 0.13.0 (2018-07-20)"},{"location":"CHANGELOG/#downloads_15","text":"Source code (zip) Source code (tar.gz)","title":"Downloads"},{"location":"CHANGELOG/#new-features_9","text":"A meaningful error message is now raised when a cross-validation generator is used with SequentialFeatureSelector . ( #377 ) The SequentialFeatureSelector now accepts custom feature names via the fit method for more interpretable feature subset reports. ( #379 ) The SequentialFeatureSelector is now also compatible with Pandas DataFrames and uses DataFrame column-names for more interpretable feature subset reports. ( #379 ) ColumnSelector now works with Pandas DataFrames columns. ( #378 by Manuel Garrido ) The ExhaustiveFeatureSelector estimator in mlxtend.feature_selection now is safely stoppable mid-process by control+c. ( #380 ) Two new functions, vectorspace_orthonormalization and vectorspace_dimensionality were added to mlxtend.math to use the Gram-Schmidt process to convert a set of linearly independent vectors into a set of orthonormal basis vectors, and to compute the dimensionality of a vectorspace, respectively. ( #382 ) mlxtend.frequent_patterns.apriori now supports pandas SparseDataFrame s to generate frequent itemsets. ( #404 via Daniel Morales ) The plot_confusion_matrix function now has the ability to show normalized confusion matrix coefficients in addition to or instead of absolute confusion matrix coefficients with or without a colorbar. The text display method has been changed so that the full range of the colormap is used. The default size is also now set based on the number of classes. Added support for merging the meta features with the original input features in StackingRegressor (via use_features_in_secondary ) like it is already supported in the other Stacking classes. ( #418 ) Added a support_only to the association_rules function, which allow constructing association rules (based on the support metric only) for cropped input DataFrames that don't contain a complete set of antecedent and consequent support values. ( #421 )","title":"New Features"},{"location":"CHANGELOG/#changes_15","text":"Itemsets generated with apriori are now frozenset s ( #393 by William Laney and #394 ) Now raises an error if a input DataFrame to apriori contains non 0, 1, True, False values. #419 )","title":"Changes"},{"location":"CHANGELOG/#bug-fixes_11","text":"Allow mlxtend estimators to be cloned via scikit-learn's clone function. ( #374 ) Fixes bug to allow the correct use of refit=False in StackingRegressor and StackingCVRegressor ( #384 and ( #385 ) by selay01 ) Allow StackingClassifier to work with sparse matrices when use_features_in_secondary=True ( #408 by Floris Hoogenbook ) Allow StackingCVRegressor to work with sparse matrices when use_features_in_secondary=True ( #416 ) Allow StackingCVClassifier to work with sparse matrices when use_features_in_secondary=True ( #417 )","title":"Bug Fixes"},{"location":"CHANGELOG/#version-0120-2018-21-04","text":"","title":"Version 0.12.0 (2018-21-04)"},{"location":"CHANGELOG/#downloads_16","text":"Source code (zip) Source code (tar.gz)","title":"Downloads"},{"location":"CHANGELOG/#new-features_10","text":"A new feature_importance_permuation function to compute the feature importance in classifiers and regressors via the permutation importance method ( #358 ) The fit method of the ExhaustiveFeatureSelector now optionally accepts **fit_params for the estimator that is used for the feature selection. ( #354 by Zach Griffith) The fit method of the SequentialFeatureSelector now optionally accepts **fit_params for the estimator that is used for the feature selection. ( #350 by Zach Griffith)","title":"New Features"},{"location":"CHANGELOG/#changes_16","text":"Replaced plot_decision_regions colors by a colorblind-friendly palette and adds contour lines for decision regions. ( #348 ) All stacking estimators now raise NonFittedErrors if any method for inference is called prior to fitting the estimator. ( #353 ) Renamed the refit parameter of both the StackingClassifier and StackingCVClassifier to use_clones to be more explicit and less misleading. ( #368 )","title":"Changes"},{"location":"CHANGELOG/#bug-fixes_12","text":"Various changes in the documentation and documentation tools to fix formatting issues ( #363 ) Fixes a bug where the StackingCVClassifier 's meta features were not stored in the original order when shuffle=True ( #370 ) Many documentation improvements, including links to the User Guides in the API docs ( #371 )","title":"Bug Fixes"},{"location":"CHANGELOG/#version-0110-2018-03-14","text":"","title":"Version 0.11.0 (2018-03-14)"},{"location":"CHANGELOG/#downloads_17","text":"Source code (zip) Source code (tar.gz)","title":"Downloads"},{"location":"CHANGELOG/#new-features_11","text":"New function implementing the resampled paired t-test procedure ( paired_ttest_resampled ) to compare the performance of two models. ( #323 ) New function implementing the k-fold paired t-test procedure ( paired_ttest_kfold_cv ) to compare the performance of two models (also called k-hold-out paired t-test). ( #324 ) New function implementing the 5x2cv paired t-test procedure ( paired_ttest_5x2cv ) proposed by Dieterrich (1998) to compare the performance of two models. ( #325 ) A refit parameter was added to stacking classes (similar to the refit parameter in the EnsembleVoteClassifier ), to support classifiers and regressors that follow the scikit-learn API but are not compatible with scikit-learn's clone function. ( #322 ) The ColumnSelector now has a drop_axis argument to use it in pipelines with CountVectorizers . ( #333 )","title":"New Features"},{"location":"CHANGELOG/#changes_17","text":"Raises an informative error message if predict or predict_meta_features is called prior to calling the fit method in StackingRegressor and StackingCVRegressor . ( #315 ) The plot_decision_regions function now automatically determines the optimal setting based on the feature dimensions and supports anti-aliasing. The old res parameter has been deprecated. ( #309 by Guillaume Poirier-Morency ) Apriori code is faster due to optimization in onehot transformation and the amount of candidates generated by the apriori algorithm. ( #327 by Jakub Smid ) The OnehotTransactions class (which is typically often used in combination with the apriori function for association rule mining) is now more memory efficient as it uses boolean arrays instead of integer arrays. In addition, the OnehotTransactions class can be now be provided with sparse argument to generate sparse representations of the onehot matrix to further improve memory efficiency. ( #328 by Jakub Smid ) The OneHotTransactions has been deprecated and replaced by the TransactionEncoder . ( #332 The plot_decision_regions function now has three new parameters, scatter_kwargs , contourf_kwargs , and scatter_highlight_kwargs , that can be used to modify the plotting style. ( #342 by James Bourbeau )","title":"Changes"},{"location":"CHANGELOG/#bug-fixes_13","text":"Fixed issue when class labels were provided to the EnsembleVoteClassifier when refit was set to false . ( #322 ) Allow arrays with 16-bit and 32-bit precision in plot_decision_regions function. ( #337 ) Fixed bug that raised an indexing error if the number of items was <= 1 when computing association rules using the conviction metric. ( #340 )","title":"Bug Fixes"},{"location":"CHANGELOG/#version-0100-2017-12-22","text":"","title":"Version 0.10.0 (2017-12-22)"},{"location":"CHANGELOG/#downloads_18","text":"Source code (zip) Source code (tar.gz)","title":"Downloads"},{"location":"CHANGELOG/#new-features_12","text":"New store_train_meta_features parameter for fit in StackingCVRegressor. if True, train meta-features are stored in self.train_meta_features_ . New pred_meta_features method for StackingCVRegressor . People can get test meta-features using this method. ( #294 via takashioya ) The new store_train_meta_features attribute and pred_meta_features method for the StackingCVRegressor were also added to the StackingRegressor , StackingClassifier , and StackingCVClassifier ( #299 & #300 ) New function ( evaluate.mcnemar_tables ) for creating multiple 2x2 contigency from model predictions arrays that can be used in multiple McNemar (post-hoc) tests or Cochran's Q or F tests, etc. ( #307 ) New function ( evaluate.cochrans_q ) for performing Cochran's Q test to compare the accuracy of multiple classifiers. ( #310 )","title":"New Features"},{"location":"CHANGELOG/#changes_18","text":"Added requirements.txt to setup.py . ( #304 via Colin Carrol )","title":"Changes"},{"location":"CHANGELOG/#bug-fixes_14","text":"Improved numerical stability for p-values computed via the the exact McNemar test ( #306 ) nose is not required to use the library ( #302 )","title":"Bug Fixes"},{"location":"CHANGELOG/#version-091-2017-11-19","text":"","title":"Version 0.9.1 (2017-11-19)"},{"location":"CHANGELOG/#downloads_19","text":"Source code (zip) Source code (tar.gz)","title":"Downloads"},{"location":"CHANGELOG/#new-features_13","text":"Added mlxtend.evaluate.bootstrap_point632_score to evaluate the performance of estimators using the .632 bootstrap. ( #283 ) New max_len parameter for the frequent itemset generation via the apriori function to allow for early stopping. ( #270 )","title":"New Features"},{"location":"CHANGELOG/#changes_19","text":"All feature index tuples in SequentialFeatureSelector or now in sorted order. ( #262 ) The SequentialFeatureSelector now runs the continuation of the floating inclusion/exclusion as described in Novovicova & Kittler (1994). Note that this didn't cause any difference in performance on any of the test scenarios but could lead to better performance in certain edge cases. ( #262 ) utils.Counter now accepts a name variable to help distinguish between multiple counters, time precision can be set with the 'precision' kwarg and the new attribute end_time holds the time the last iteration completed. ( #278 via Mathew Savage )","title":"Changes"},{"location":"CHANGELOG/#bug-fixes_15","text":"Fixed an deprecation error that occured with McNemar test when using SciPy 1.0. ( #283 )","title":"Bug Fixes"},{"location":"CHANGELOG/#version-090-2017-10-21","text":"","title":"Version 0.9.0 (2017-10-21)"},{"location":"CHANGELOG/#downloads_20","text":"Source code (zip) Source code (tar.gz)","title":"Downloads"},{"location":"CHANGELOG/#new-features_14","text":"Added evaluate.permutation_test , a permutation test for hypothesis testing (or A/B testing) to test if two samples come from the same distribution. Or in other words, a procedure to test the null hypothesis that that two groups are not significantly different (e.g., a treatment and a control group). ( #250 ) Added 'leverage' and 'conviction as evaluation metrics to the frequent_patterns.association_rules function. ( #246 & #247 ) Added a loadings_ attribute to PrincipalComponentAnalysis to compute the factor loadings of the features on the principal components. ( #251 ) Allow grid search over classifiers/regressors in ensemble and stacking estimators. ( #259 ) New make_multiplexer_dataset function that creates a dataset generated by a n-bit Boolean multiplexer for evaluating supervised learning algorithms. ( #263 ) Added a new BootstrapOutOfBag class, an implementation of the out-of-bag bootstrap to evaluate supervised learning algorithms. ( #265 ) The parameters for StackingClassifier , StackingCVClassifier , StackingRegressor , StackingCVRegressor , and EnsembleVoteClassifier can now be tuned using scikit-learn's GridSearchCV ( #254 via James Bourbeau )","title":"New Features"},{"location":"CHANGELOG/#changes_20","text":"The 'support' column returned by frequent_patterns.association_rules was changed to compute the support of \"antecedant union consequent\", and new antecedant support' and 'consequent support' column were added to avoid ambiguity. ( #245 ) Allow the OnehotTransactions to be cloned via scikit-learn's clone function, which is required by e.g., scikit-learn's FeatureUnion or GridSearchCV (via Iaroslav Shcherbatyi ). ( #249 )","title":"Changes"},{"location":"CHANGELOG/#bug-fixes_16","text":"Fix issues with self._init_time parameter in _IterativeModel subclasses. ( #256 ) Fix imprecision bug that occurred in plot_ecdf when run on Python 2.7. ( 264 ) The vectors from SVD in PrincipalComponentAnalysis are now being scaled so that the eigenvalues via solver='eigen' and solver='svd' now store eigenvalues that have the same magnitudes. ( #251 )","title":"Bug Fixes"},{"location":"CHANGELOG/#version-080-2017-09-09","text":"","title":"Version 0.8.0 (2017-09-09)"},{"location":"CHANGELOG/#downloads_21","text":"Source code (zip) Source code (tar.gz)","title":"Downloads"},{"location":"CHANGELOG/#new-features_15","text":"Added a mlxtend.evaluate.bootstrap that implements the ordinary nonparametric bootstrap to bootstrap a single statistic (for example, the mean. median, R^2 of a regression fit, and so forth) #232 SequentialFeatureSelecor 's k_features now accepts a string argument \"best\" or \"parsimonious\" for more \"automated\" feature selection. For instance, if \"best\" is provided, the feature selector will return the feature subset with the best cross-validation performance. If \"parsimonious\" is provided as an argument, the smallest feature subset that is within one standard error of the cross-validation performance will be selected. #238","title":"New Features"},{"location":"CHANGELOG/#changes_21","text":"SequentialFeatureSelector now uses np.nanmean over normal mean to support scorers that may return np.nan #211 (via mrkaiser ) The skip_if_stuck parameter was removed from SequentialFeatureSelector in favor of a more efficient implementation comparing the conditional inclusion/exclusion results (in the floating versions) to the performances of previously sampled feature sets that were cached #237 ExhaustiveFeatureSelector was modified to consume substantially less memory #195 (via Adam Erickson )","title":"Changes"},{"location":"CHANGELOG/#bug-fixes_17","text":"Fixed a bug where the SequentialFeatureSelector selected a feature subset larger than then specified via the k_features tuple max-value #213","title":"Bug Fixes"},{"location":"CHANGELOG/#version-070-2017-06-22","text":"","title":"Version 0.7.0 (2017-06-22)"},{"location":"CHANGELOG/#downloads_22","text":"Source code (zip) Source code (tar.gz)","title":"Downloads"},{"location":"CHANGELOG/#new-features_16","text":"New mlxtend.plotting.ecdf function for plotting empirical cumulative distribution functions ( #196 ). New StackingCVRegressor for stacking regressors with out-of-fold predictions to prevent overfitting ( #201 via Eike Dehling ).","title":"New Features"},{"location":"CHANGELOG/#changes_22","text":"The TensorFlow estimator have been removed from mlxtend, since TensorFlow has now very convenient ways to build on estimators, which render those implementations obsolete. plot_decision_regions now supports plotting decision regions for more than 2 training features #189 , via James Bourbeau ). Parallel execution in mlxtend.feature_selection.SequentialFeatureSelector and mlxtend.feature_selection.ExhaustiveFeatureSelector is now performed over different feature subsets instead of the different cross-validation folds to better utilize machines with multiple processors if the number of features is large ( #193 , via @whalebot-helmsman ). Raise meaningful error messages if pandas DataFrame s or Python lists of lists are fed into the StackingCVClassifer as a fit arguments ( 198 ). The n_folds parameter of the StackingCVClassifier was changed to cv and can now accept any kind of cross validation technique that is available from scikit-learn. For example, StackingCVClassifier(..., cv=StratifiedKFold(n_splits=3)) or StackingCVClassifier(..., cv=GroupKFold(n_splits=3)) ( #203 , via Konstantinos Paliouras ).","title":"Changes"},{"location":"CHANGELOG/#bug-fixes_18","text":"SequentialFeatureSelector now correctly accepts a None argument for the scoring parameter to infer the default scoring metric from scikit-learn classifiers and regressors ( #171 ). The plot_decision_regions function now supports pre-existing axes objects generated via matplotlib's plt.subplots . ( #184 , see example ) Made math.num_combinations and math.num_permutations numerically stable for large numbers of combinations and permutations ( #200 ).","title":"Bug Fixes"},{"location":"CHANGELOG/#version-060-2017-03-18","text":"","title":"Version 0.6.0 (2017-03-18)"},{"location":"CHANGELOG/#downloads_23","text":"Source code (zip) Source code (tar.gz)","title":"Downloads"},{"location":"CHANGELOG/#new-features_17","text":"An association_rules function is implemented that allows to generate rules based on a list of frequent itemsets (via Joshua Goerner ).","title":"New Features"},{"location":"CHANGELOG/#changes_23","text":"Adds a black edgecolor to plots via plotting.plot_decision_regions to make markers more distinguishable from the background in matplotlib>=2.0 . The association submodule was renamed to frequent_patterns .","title":"Changes"},{"location":"CHANGELOG/#bug-fixes_19","text":"The DataFrame index of apriori results are now unique and ordered. Fixed typos in autompg and wine datasets (via James Bourbeau ).","title":"Bug Fixes"},{"location":"CHANGELOG/#version-051-2017-02-14","text":"","title":"Version 0.5.1 (2017-02-14)"},{"location":"CHANGELOG/#downloads_24","text":"Source code (zip) Source code (tar.gz)","title":"Downloads"},{"location":"CHANGELOG/#new-features_18","text":"The EnsembleVoteClassifier has a new refit attribute that prevents refitting classifiers if refit=False to save computational time. Added a new lift_score function in evaluate to compute lift score (via Batuhan Bardak ). StackingClassifier and StackingRegressor support multivariate targets if the underlying models do (via kernc ). StackingClassifier has a new use_features_in_secondary attribute like StackingCVClassifier .","title":"New Features"},{"location":"CHANGELOG/#changes_24","text":"Changed default verbosity level in SequentialFeatureSelector to 0 The EnsembleVoteClassifier now raises a NotFittedError if the estimator wasn't fit before calling predict . (via Anton Loss ) Added new TensorFlow variable initialization syntax to guarantee compatibility with TensorFlow 1.0","title":"Changes"},{"location":"CHANGELOG/#bug-fixes_20","text":"Fixed wrong default value for k_features in SequentialFeatureSelector Cast selected feature subsets in the SequentialFeautureSelector as sets to prevent the iterator from getting stuck if the k_idx are different permutations of the same combination (via Zac Wellmer ). Fixed an issue with learning curves that caused the performance metrics to be reversed (via ipashchenko ) Fixed a bug that could occur in the SequentialFeatureSelector if there are similarly-well performing subsets in the floating variants (via Zac Wellmer ).","title":"Bug Fixes"},{"location":"CHANGELOG/#version-050-2016-11-09","text":"","title":"Version 0.5.0 (2016-11-09)"},{"location":"CHANGELOG/#downloads_25","text":"Source code (zip) Source code (tar.gz)","title":"Downloads"},{"location":"CHANGELOG/#new-features_19","text":"New ExhaustiveFeatureSelector estimator in mlxtend.feature_selection for evaluating all feature combinations in a specified range The StackingClassifier has a new parameter average_probas that is set to True by default to maintain the current behavior. A deprecation warning was added though, and it will default to False in future releases (0.6.0); average_probas=False will result in stacking of the level-1 predicted probabilities rather than averaging these. New StackingCVClassifier estimator in 'mlxtend.classifier' for implementing a stacking ensemble that uses cross-validation techniques for training the meta-estimator to avoid overfitting ( Reiichiro Nakano ) New OnehotTransactions encoder class added to the preprocessing submodule for transforming transaction data into a one-hot encoded array The SequentialFeatureSelector estimator in mlxtend.feature_selection now is safely stoppable mid-process by control+c, and deprecated print_progress in favor of a more tunable verbose parameter ( Will McGinnis ) New apriori function in association to extract frequent itemsets from transaction data for association rule mining New checkerboard_plot function in plotting to plot checkerboard tables / heat maps New mcnemar_table and mcnemar functions in evaluate to compute 2x2 contingency tables and McNemar's test","title":"New Features"},{"location":"CHANGELOG/#changes_25","text":"All plotting functions have been moved to mlxtend.plotting for compatibility reasons with continuous integration services and to make the installation of matplotlib optional for users of mlxtend 's core functionality Added a compatibility layer for scikit-learn 0.18 using the new model_selection module while maintaining backwards compatibility to scikit-learn 0.17.","title":"Changes"},{"location":"CHANGELOG/#bug-fixes_21","text":"mlxtend.plotting.plot_decision_regions now draws decision regions correctly if more than 4 class labels are present Raise AttributeError in plot_decision_regions when the X_higlight argument is a 1D array ( chkoar )","title":"Bug Fixes"},{"location":"CHANGELOG/#version-042-2016-08-24","text":"","title":"Version 0.4.2 (2016-08-24)"},{"location":"CHANGELOG/#downloads_26","text":"Source code (zip) Source code (tar.gz) PDF documentation","title":"Downloads"},{"location":"CHANGELOG/#new-features_20","text":"Added preprocessing.CopyTransformer , a mock class that returns copies of imput arrays via transform and fit_transform","title":"New Features"},{"location":"CHANGELOG/#changes_26","text":"Added AppVeyor to CI to ensure MS Windows compatibility Dataset are now saved as compressed .txt or .csv files rather than being imported as Python objects feature_selection.SequentialFeatureSelector now supports the selection of k_features using a tuple to specify a \"min-max\" k_features range Added \"SVD solver\" option to the PrincipalComponentAnalysis Raise a AttributeError with \"not fitted\" message in SequentialFeatureSelector if transform or get_metric_dict are called prior to fit Use small, positive bias units in TfMultiLayerPerceptron 's hidden layer(s) if the activations are ReLUs in order to avoid dead neurons Added an optional clone_estimator parameter to the SequentialFeatureSelector that defaults to True , avoiding the modification of the original estimator objects More rigorous type and shape checks in the evaluate.plot_decision_regions function DenseTransformer now doesn't raise and error if the input array is not sparse API clean-up using scikit-learn's BaseEstimator as parent class for feature_selection.ColumnSelector","title":"Changes"},{"location":"CHANGELOG/#bug-fixes_22","text":"Fixed a problem when a tuple-range was provided as argument to the SequentialFeatureSelector 's k_features parameter and the scoring metric was more negative than -1 (e.g., as in scikit-learn's MSE scoring function) (wahutch](https://github.com/wahutch)) Fixed an AttributeError issue when verbose > 1 in StackingClassifier Fixed a bug in classifier.SoftmaxRegression where the mean values of the offsets were used to update the bias units rather than their sum Fixed rare bug in MLP _layer_mapping functions that caused a swap between the random number generation seed when initializing weights and biases","title":"Bug Fixes"},{"location":"CHANGELOG/#version-041-2016-05-01","text":"","title":"Version 0.4.1 (2016-05-01)"},{"location":"CHANGELOG/#downloads_27","text":"Source code (zip) Source code (tar.gz) PDF documentation","title":"Downloads"},{"location":"CHANGELOG/#new-features_21","text":"New TensorFlow estimator for Linear Regression ( tf_regressor.TfLinearRegression ) New k-means clustering estimator ( cluster.Kmeans ) New TensorFlow k-means clustering estimator ( tf_cluster.Kmeans )","title":"New Features"},{"location":"CHANGELOG/#changes_27","text":"Due to refactoring of the estimator classes, the init_weights parameter of the fit methods was globally renamed to init_params Overall performance improvements of estimators due to code clean-up and refactoring Added several additional checks for correct array types and more meaningful exception messages Added optional dropout to the tf_classifier.TfMultiLayerPerceptron classifier for regularization Added an optional decay parameter to the tf_classifier.TfMultiLayerPerceptron classifier for adaptive learning via an exponential decay of the learning rate eta Replaced old NeuralNetMLP by more streamlined MultiLayerPerceptron ( classifier.MultiLayerPerceptron ); now also with softmax in the output layer and categorical cross-entropy loss. Unified init_params parameter for fit functions to continue training where the algorithm left off (if supported)","title":"Changes"},{"location":"CHANGELOG/#version-040-2016-04-09","text":"","title":"Version 0.4.0 (2016-04-09)"},{"location":"CHANGELOG/#new-features_22","text":"New TfSoftmaxRegression classifier using Tensorflow ( tf_classifier.TfSoftmaxRegression ) New SoftmaxRegression classifier ( classifier.SoftmaxRegression ) New TfMultiLayerPerceptron classifier using Tensorflow ( tf_classifier.TfMultiLayerPerceptron ) New StackingRegressor ( regressor.StackingRegressor ) New StackingClassifier ( classifier.StackingClassifier ) New function for one-hot encoding of class labels ( preprocessing.one_hot ) Added GridSearch support to the SequentialFeatureSelector ( feature_selection/.SequentialFeatureSelector ) evaluate.plot_decision_regions improvements: Function now handles class y-class labels correctly if array is of type float Correct handling of input arguments markers and colors Accept an existing Axes via the ax argument New print_progress parameter for all generalized models and multi-layer neural networks for printing time elapsed, ETA, and the current cost of the current epoch Minibatch learning for classifier.LogisticRegression , classifier.Adaline , and regressor.LinearRegression plus streamlined API New Principal Component Analysis class via mlxtend.feature_extraction.PrincipalComponentAnalysis New RBF Kernel Principal Component Analysis class via mlxtend.feature_extraction.RBFKernelPCA New Linear Discriminant Analysis class via mlxtend.feature_extraction.LinearDiscriminantAnalysis","title":"New Features"},{"location":"CHANGELOG/#changes_28","text":"The column parameter in mlxtend.preprocessing.standardize now defaults to None to standardize all columns more conveniently","title":"Changes"},{"location":"CHANGELOG/#version-030-2016-01-31","text":"","title":"Version 0.3.0 (2016-01-31)"},{"location":"CHANGELOG/#downloads_28","text":"Source code (zip) Source code (tar.gz)","title":"Downloads"},{"location":"CHANGELOG/#new-features_23","text":"Added a progress bar tracker to classifier.NeuralNetMLP Added a function to score predicted vs. target class labels evaluate.scoring Added confusion matrix functions to create ( evaluate.confusion_matrix ) and plot ( evaluate.plot_confusion_matrix ) confusion matrices New style parameter and improved axis scaling in mlxtend.evaluate.plot_learning_curves Added loadlocal_mnist to mlxtend.data for streaming MNIST from a local byte files into numpy arrays New NeuralNetMLP parameters: random_weights , shuffle_init , shuffle_epoch New SFS features such as the generation of pandas DataFrame results tables and plotting functions (with confidence intervals, standard deviation, and standard error bars) Added support for regression estimators in SFS Added Boston housing dataset New shuffle parameter for classifier.NeuralNetMLP","title":"New Features"},{"location":"CHANGELOG/#changes_29","text":"The mlxtend.preprocessing.standardize function now optionally returns the parameters, which are estimated from the array, for re-use. A further improvement makes the standardize function smarter in order to avoid zero-division errors Cosmetic improvements to the evaluate.plot_decision_regions function such as hiding plot axes Renaming of classifier.EnsembleClassfier to classifier.EnsembleVoteClassifier Improved random weight initialization in Perceptron , Adaline , LinearRegression , and LogisticRegression Changed learning parameter of mlxtend.classifier.Adaline to solver and added \"normal equation\" as closed-form solution solver Hide y-axis labels in mlxtend.evaluate.plot_decision_regions in 1 dimensional evaluations Sequential Feature Selection algorithms were unified into a single SequentialFeatureSelector class with parameters to enable floating selection and toggle between forward and backward selection. Stratified sampling of MNIST (now 500x random samples from each of the 10 digit categories) Renaming mlxtend.plotting to mlxtend.general_plotting in order to distinguish general plotting function from specialized utility function such as evaluate.plot_decision_regions","title":"Changes"},{"location":"CHANGELOG/#version-029-2015-07-14","text":"","title":"Version 0.2.9 (2015-07-14)"},{"location":"CHANGELOG/#downloads_29","text":"Source code (zip) Source code (tar.gz)","title":"Downloads"},{"location":"CHANGELOG/#new-features_24","text":"Sequential Feature Selection algorithms: SFS, SFFS, SBS, and SFBS","title":"New Features"},{"location":"CHANGELOG/#changes_30","text":"Changed regularization & lambda parameters in LogisticRegression to single parameter l2_lambda","title":"Changes"},{"location":"CHANGELOG/#version-028-2015-06-27","text":"API changes: mlxtend.sklearn.EnsembleClassifier -> mlxtend.classifier.EnsembleClassifier mlxtend.sklearn.ColumnSelector -> mlxtend.feature_selection.ColumnSelector mlxtend.sklearn.DenseTransformer -> mlxtend.preprocessing.DenseTransformer mlxtend.pandas.standardizing -> mlxtend.preprocessing.standardizing mlxtend.pandas.minmax_scaling -> mlxtend.preprocessing.minmax_scaling mlxtend.matplotlib -> mlxtend.plotting Added momentum learning parameter (alpha coefficient) to mlxtend.classifier.NeuralNetMLP . Added adaptive learning rate (decrease constant) to mlxtend.classifier.NeuralNetMLP . mlxtend.pandas.minmax_scaling became mlxtend.preprocessing.minmax_scaling and also supports NumPy arrays now mlxtend.pandas.standardizing became mlxtend.preprocessing.standardizing and now supports both NumPy arrays and pandas DataFrames; also, now ddof parameters to set the degrees of freedom when calculating the standard deviation","title":"Version 0.2.8 (2015-06-27)"},{"location":"CHANGELOG/#version-027-2015-06-20","text":"Added multilayer perceptron (feedforward artificial neural network) classifier as mlxtend.classifier.NeuralNetMLP . Added 5000 labeled trainingsamples from the MNIST handwritten digits dataset to mlxtend.data","title":"Version 0.2.7 (2015-06-20)"},{"location":"CHANGELOG/#version-026-2015-05-08","text":"Added ordinary least square regression using different solvers (gradient and stochastic gradient descent, and the closed form solution (normal equation) Added option for random weight initialization to logistic regression classifier and updated l2 regularization Added wine dataset to mlxtend.data Added invert_axes parameter mlxtend.matplotlib.enrichtment_plot to optionally plot the \"Count\" on the x-axis New verbose parameter for mlxtend.sklearn.EnsembleClassifier by Alejandro C. Bahnsen Added mlxtend.pandas.standardizing to standardize columns in a Pandas DataFrame Added parameters linestyles and markers to mlxtend.matplotlib.enrichment_plot mlxtend.regression.lin_regplot automatically adds np.newaxis and works w. python lists Added tokenizers: mlxtend.text.extract_emoticons and mlxtend.text.extract_words_and_emoticons","title":"Version 0.2.6 (2015-05-08)"},{"location":"CHANGELOG/#version-025-2015-04-17","text":"Added Sequential Backward Selection (mlxtend.sklearn.SBS) Added X_highlight parameter to mlxtend.evaluate.plot_decision_regions for highlighting test data points. Added mlxtend.regression.lin_regplot to plot the fitted line from linear regression. Added mlxtend.matplotlib.stacked_barplot to conveniently produce stacked barplots using pandas DataFrame s. Added mlxtend.matplotlib.enrichment_plot","title":"Version 0.2.5 (2015-04-17)"},{"location":"CHANGELOG/#version-024-2015-03-15","text":"Added scoring to mlxtend.evaluate.learning_curves (by user pfsq) Fixed setup.py bug caused by the missing README.html file matplotlib.category_scatter for pandas DataFrames and Numpy arrays","title":"Version 0.2.4 (2015-03-15)"},{"location":"CHANGELOG/#version-023-2015-03-11","text":"Added Logistic regression Gradient descent and stochastic gradient descent perceptron was changed to Adaline (Adaptive Linear Neuron) Perceptron and Adaline for {0, 1} classes Added mlxtend.preprocessing.shuffle_arrays_unison function to shuffle one or more NumPy arrays. Added shuffle and random seed parameter to stochastic gradient descent classifier. Added rstrip parameter to mlxtend.file_io.find_filegroups to allow trimming of base names. Added ignore_substring parameter to mlxtend.file_io.find_filegroups and find_files . Replaced .rstrip in mlxtend.file_io.find_filegroups with more robust regex. Gridsearch support for mlxtend.sklearn.EnsembleClassifier","title":"Version 0.2.3 (2015-03-11)"},{"location":"CHANGELOG/#version-022-2015-03-01","text":"Improved robustness of EnsembleClassifier. Extended plot_decision_regions() functionality for plotting 1D decision boundaries. Function matplotlib.plot_decision_regions was reorganized to evaluate.plot_decision_regions . evaluate.plot_learning_curves() function added. Added Rosenblatt, gradient descent, and stochastic gradient descent perceptrons.","title":"Version 0.2.2 (2015-03-01)"},{"location":"CHANGELOG/#version-021-2015-01-20","text":"Added mlxtend.pandas.minmax_scaling - a function to rescale pandas DataFrame columns. Slight update to the EnsembleClassifier interface (additional voting parameter) Fixed EnsembleClassifier to return correct class labels if class labels are not integers from 0 to n. Added new matplotlib function to plot decision regions of classifiers.","title":"Version 0.2.1 (2015-01-20)"},{"location":"CHANGELOG/#version-020-2015-01-13","text":"Improved mlxtend.text.generalize_duplcheck to remove duplicates and prevent endless looping issue. Added recursive search parameter to mlxtend.file_io.find_files. Added check_ext parameter mlxtend.file_io.find_files to search based on file extensions. Default parameter to ignore invisible files for mlxtend.file_io.find. Added transform and fit_transform to the EnsembleClassifier . Added mlxtend.file_io.find_filegroups function.","title":"Version 0.2.0 (2015-01-13)"},{"location":"CHANGELOG/#version-019-2015-01-10","text":"Implemented scikit-learn EnsembleClassifier (majority voting rule) class.","title":"Version 0.1.9 (2015-01-10)"},{"location":"CHANGELOG/#version-018-2015-01-07","text":"Improvements to mlxtend.text.generalize_names to handle certain Dutch last name prefixes (van, van der, de, etc.). Added mlxtend.text.generalize_name_duplcheck function to apply mlxtend.text.generalize_names function to a pandas DataFrame without creating duplicates.","title":"Version 0.1.8 (2015-01-07)"},{"location":"CHANGELOG/#version-017-2015-01-07","text":"Added text utilities with name generalization function. Added and file_io utilities.","title":"Version 0.1.7 (2015-01-07)"},{"location":"CHANGELOG/#version-016-2015-01-04","text":"Added combinations and permutations estimators.","title":"Version 0.1.6 (2015-01-04)"},{"location":"CHANGELOG/#version-015-2014-12-11","text":"Added DenseTransformer for pipelines and grid search.","title":"Version 0.1.5 (2014-12-11)"},{"location":"CHANGELOG/#version-014-2014-08-20","text":"mean_centering function is now a Class that creates MeanCenterer objects that can be used to fit data via the fit method, and center data at the column means via the transform and fit_transform method.","title":"Version 0.1.4 (2014-08-20)"},{"location":"CHANGELOG/#version-013-2014-08-19","text":"Added preprocessing module and mean_centering function.","title":"Version 0.1.3 (2014-08-19)"},{"location":"CHANGELOG/#version-012-2014-08-19","text":"Added matplotlib utilities and remove_borders function.","title":"Version 0.1.2 (2014-08-19)"},{"location":"CHANGELOG/#version-011-2014-08-13","text":"Simplified code for ColumnSelector.","title":"Version 0.1.1 (2014-08-13)"},{"location":"CONTRIBUTING/","text":"How to Contribute I would be very happy about any kind of contributions that help to improve and extend the functionality of mlxtend. Quick Contributor Checklist This is a quick checklist about the different steps of a typical contribution to mlxtend (and other open source projects). Consider copying this list to a local text file (or the issue tracker) and checking off items as you go. 1) Making and testing code changes: [ ] Open a new \"issue\" on GitHub to discuss the new feature / bug fix [ ] Fork the mlxtend repository from GitHub (if not already done earlier) [ ] Create and check out a new topic branch (please don't make modifications in the master branch) [ ] Implement the new feature or apply the bug-fix [ ] Add appropriate unit test functions in mlxtend/*/tests [ ] Run PYTHONPATH='.' pytest ./mlxtend -sv and make sure that all unit tests pass [ ] Make sure the newly implemented feature has good test coverage: python -m pip install coverage # test all: # coverage run --source=mlxtend --branch -m pytest . coverage run --source=mlxtend --branch -m pytest mlxtend/<insert_path> coverage html [ ] Modify documentation in the appropriate location under mlxtend/docs/sources/ [ ] Add a note about the modification/contribution to the ./docs/sources/changelog.md file 2) Checking code style: When you check in a PR, mlxtend will run code style checks via flak8 and black. To make the contributor experience easier, we recommend you check the code style locally before pushing it to the repository. This way it is less likely that the automated checkers will complain and prompt you to make fixes. There are two ways you can do this: Option A : Running the tools manually [ ] Check for style issues by running flake8 ./mlxtend (you may want to run pytest again after you made modifications to the code) [ ] We recommend using black to format the code automatically according to recommended style changes. After installing black , you can do this via black [source_file_or_directory] [ ] Run isort which will sort the imports alphabetically. We recommend the following command: isort -p mlxtend --line-length 88 --multi-line 3 --profile black mypythonfile.py Option B : Using pre-commit hooks (recommended) The pre-commit hooks for mlxtend will check your code via flake8 , black , and isort automatically before you make a git commit . You can read more about pre-commit hooks here . [ ] Install the pre-commit package via pip install pre-commit . [ ] In the mlxtend folder, run pre-commit install (you only have to do it once). 3) Submitting your code [ ] Push the topic branch to the server and create a pull request. [ ] Check the automated tests passed. [ ] The automatic PEP8 / black integrations may prompt you to modify the code stylistically. It would be nice if you could apply the suggested changes. Tips for Contributors Getting Started - Creating a New Issue and Forking the Repository If you don't have a GitHub account, yet, please create one to contribute to this project. Please submit a ticket for your issue to discuss the fix or new feature before too much time and effort is spent for the implementation. Fork the mlxtend repository from the GitHub web interface. Clone the mlxtend repository to your local machine by executing git clone https://github.com/<your_username>/mlxtend.git Syncing an Existing Fork If you already forked mlxtend earlier, you can bring you \"Fork\" up to date with the master branch as follows: 1. Configuring a remote that points to the upstream repository on GitHub List the current configured remote repository of your fork by executing $ git remote -v If you see something like origin https://github.com/<your username>/mlxtend.git (fetch) origin https://github.com/<your username>/mlxtend.git (push) you need to specify a new remote upstream repository via $ git remote add upstream https://github.com/rasbt/mlxtend.git Now, verify the new upstream repository you've specified for your fork by executing $ git remote -v You should see following output if everything is configured correctly: origin https://github.com/<your username>/mlxtend.git (fetch) origin https://github.com/<your username>/mlxtend.git (push) upstream https://github.com/rasbt/mlxtend.git (fetch) upstream https://github.com/rasbt/mlxtend.git (push) 2. Syncing your Fork First, fetch the updates of the original project's master branch by executing: $ git fetch upstream You should see the following output remote: Counting objects: xx, done. remote: Compressing objects: 100% (xx/xx), done. remote: Total xx (delta xx), reused xx (delta x) Unpacking objects: 100% (xx/xx), done. From https://github.com/rasbt/mlxtend * [new branch] master -> upstream/master This means that the commits to the rasbt/mlxtend master branch are now stored in the local branch upstream/master . If you are not already on your local project's master branch, execute $ git checkout master Finally, merge the changes in upstream/master to your local master branch by executing $ git merge upstream/master which will give you an output that looks similar to Updating xxx...xxx Fast-forward SOME FILE1 | 12 +++++++ SOME FILE2 | 10 +++++++ 2 files changed, 22 insertions(+), *The Main Workflow - Making Changes in a New Topic Branch Listed below are the 9 typical steps of a contribution. 1. Discussing the Feature or Modification Before you start coding, please discuss the new feature, bugfix, or other modification to the project on the project's issue tracker . Before you open a \"new issue,\" please do a quick search to see if a similar issue has been submitted already. 2. Creating a new feature branch Please avoid working directly on the master branch but create a new feature branch: $ git branch <new_feature> Switch to the new feature branch by executing $ git checkout <new_feature> 3. Developing the new feature / bug fix Now it's time to modify existing code or to contribute new code to the project. 4. Testing your code Add the respective unit tests and check if they pass: $ PYTHONPATH='.' pytest ./mlxtend ---with-coverage 5. Documenting changes Please add an entry to the mlxtend/docs/sources/changelog.md file. If it is a new feature, it would also be nice if you could update the documentation in appropriate location in mlxtend/sources . 6. Committing changes When you are ready to commit the changes, please provide a meaningful commit message: $ git add <modifies_files> # or `git add .` $ git commit -m '<meaningful commit message>' 7. Optional: squashing commits If you made multiple smaller commits, it would be nice if you could group them into a larger, summarizing commit. First, list your recent commit via Note Due to the improved GitHub UI, this is no longer necessary/encouraged. $ git log which will list the commits from newest to oldest in the following format by default: commit 046e3af8a9127df8eac879454f029937c8a31c41 Author: rasbt <mail@sebastianraschka.com> Date: Tue Nov 24 03:46:37 2015 -0500 fixed setup.py commit c3c00f6ba0e8f48bbe1c9081b8ae3817e57ecc5c Author: rasbt <mail@sebastianraschka.com> Date: Tue Nov 24 03:04:39 2015 -0500 documented feature x commit d87934fe8726c46f0b166d6290a3bf38915d6e75 Author: rasbt <mail@sebastianraschka.com> Date: Tue Nov 24 02:44:45 2015 -0500 added support for feature x Assuming that it would make sense to group these 3 commits into one, we can execute $ git rebase -i HEAD~3 which will bring our default git editor with the following contents: pick d87934f added support for feature x pick c3c00f6 documented feature x pick 046e3af fixed setup.py Since c3c00f6 and 046e3af are related to the original commit of feature x , let's keep the d87934f and squash the 2 following commits into this initial one by changes the lines to pick d87934f added support for feature x squash c3c00f6 documented feature x squash 046e3af fixed setup.py Now, save the changes in your editor. Now, quitting the editor will apply the rebase changes, and the editor will open a second time, prompting you to enter a new commit message. In this case, we could enter support for feature x to summarize the contributions. 8. Uploading changes Push your changes to a topic branch to the git server by executing: $ git push origin <feature_branch> 9. Submitting a pull request Go to your GitHub repository online, select the new feature branch, and submit a new pull request: Notes for Developers Building the documentation The documentation is built via MkDocs ; to ensure that the documentation is rendered correctly, you can view the documentation locally by executing mkdocs serve from the mlxtend/docs directory. For example, ~/github/mlxtend/docs$ mkdocs serve 1. Building the API documentation To build the API documentation, navigate to mlxtend/docs and execute the make_api.py file from this directory via ~/github/mlxtend/docs$ python make_api.py This should place the API documentation into the correct directories into the two directories: mlxtend/docs/sources/api_modules mlxtend/docs/sources/api_subpackes 2. Editing the User Guide The documents containing code examples for the \"User Guide\" are generated from IPython Notebook files. In order to convert a IPython notebook file to markdown after editing, please follow the following steps: Modify or edit the existing notebook. Execute all cells in the current notebook and make sure that no errors occur. Convert the notebook to markdown using the ipynb2markdown.py converter ~/github/mlxtend/docs$ python ipynb2markdown.py --ipynb ./sources/user_guide/subpackage/notebookname.ipynb Note If you are adding a new document, please also include it in the pages section in the mlxtend/docs/mkdocs.yml file. 3. Building static HTML files of the documentation First, please check the documenation via localhost (https://127.0.0.1:8000/): ~/github/mlxtend/docs$ mkdocs serve Next, build the static HTML files of the mlxtend documentation via ~/github/mlxtend/docs$ mkdocs build --clean To deploy the documentation, execute ~/github/mlxtend/docs$ mkdocs gh-deploy --clean 4. Generate a PDF of the documentation To generate a PDF version of the documentation, simply cd into the mlxtend/docs directory and execute: python md2pdf.py Uploading a new version to PyPI 1. Creating a new testing environment Assuming we are using conda , create a new python environment via $ conda create -n 'mlxtend-testing' python=3 numpy scipy pandas Next, activate the environment by executing $ source activate mlxtend-testing 2. Installing the package from local files Test the installation by executing the following from within the mlxtend main directory: $ pip install . -e the --record files.txt flag will create a files.txt file listing the locations where these files will be installed. Try to import the package to see if it works, for example, by executing $ python -c 'import mlxtend; print(mlxtend.__file__)' If everything seems to be fine, remove the installation via $ cat files.txt | xargs rm -rf ; rm files.txt Next, test if pip is able to install the packages. First, navigate to a different directory, and from there, install the package: $ pip uninstall $ pip install mlxtend and uninstall it again $ pip uninstall mlxtend 3. Deploying the package Consider deploying the package to the PyPI test server first. The setup instructions can be found here . First , install Twine if you don't have it already installed. E.g., use the following to install all recommended packages: $ python -m pip install twine build Second , create the distribution. This by default creates an sdist and wheel in the ./dist directory. $ python -m build Install the wheel and sdist to make sure they work. The distributions file names will change with each version. python -m pip install ./dist/mlxtend-0.23.0.dev0.tar.gz --force-reinstall python -m pip install ./dist/mlxtend-0.23.0.dev0-py3-none-any.whl --force-reinstall python -m pip uninstall mlxtend Third , upload the packages to the test server: $ twine upload --repository-url https://upload.pypi.org/legacy dist/ Then, install it and see if it works: $ pip install -i https://testpypi.python.org/pypi mlxtend Next, uninstall it again as follows: $ pip uninstall mlxtend Fourth , after this dry-run succeeded, repeat this process using the \"real\" PyPI: $ python -m twine upload dist/* 4. Removing the virtual environment Finally, to cleanup our local drive, remove the virtual testing environment via $ conda remove --name 'mlxtend-testing' --all Note : if you get an error like HTTPError: 403 Forbidden from https://upload.pypi.org/legacy/ make sure you have an up to date version of twine installed (helped me to uninstall in conda and install via pip.) 5. Updating the conda-forge recipe Once a new version of mlxtend has been uploaded to PyPI, update the conda-forge build recipe at https://github.com/conda-forge/mlxtend-feedstock by changing the version number in the recipe/meta.yaml file appropriately.","title":"How To Contribute"},{"location":"CONTRIBUTING/#how-to-contribute","text":"I would be very happy about any kind of contributions that help to improve and extend the functionality of mlxtend.","title":"How to Contribute"},{"location":"CONTRIBUTING/#quick-contributor-checklist","text":"This is a quick checklist about the different steps of a typical contribution to mlxtend (and other open source projects). Consider copying this list to a local text file (or the issue tracker) and checking off items as you go.","title":"Quick Contributor Checklist"},{"location":"CONTRIBUTING/#1-making-and-testing-code-changes","text":"[ ] Open a new \"issue\" on GitHub to discuss the new feature / bug fix [ ] Fork the mlxtend repository from GitHub (if not already done earlier) [ ] Create and check out a new topic branch (please don't make modifications in the master branch) [ ] Implement the new feature or apply the bug-fix [ ] Add appropriate unit test functions in mlxtend/*/tests [ ] Run PYTHONPATH='.' pytest ./mlxtend -sv and make sure that all unit tests pass [ ] Make sure the newly implemented feature has good test coverage: python -m pip install coverage # test all: # coverage run --source=mlxtend --branch -m pytest . coverage run --source=mlxtend --branch -m pytest mlxtend/<insert_path> coverage html [ ] Modify documentation in the appropriate location under mlxtend/docs/sources/ [ ] Add a note about the modification/contribution to the ./docs/sources/changelog.md file","title":"1) Making and testing code changes:"},{"location":"CONTRIBUTING/#2-checking-code-style","text":"When you check in a PR, mlxtend will run code style checks via flak8 and black. To make the contributor experience easier, we recommend you check the code style locally before pushing it to the repository. This way it is less likely that the automated checkers will complain and prompt you to make fixes. There are two ways you can do this: Option A : Running the tools manually [ ] Check for style issues by running flake8 ./mlxtend (you may want to run pytest again after you made modifications to the code) [ ] We recommend using black to format the code automatically according to recommended style changes. After installing black , you can do this via black [source_file_or_directory] [ ] Run isort which will sort the imports alphabetically. We recommend the following command: isort -p mlxtend --line-length 88 --multi-line 3 --profile black mypythonfile.py Option B : Using pre-commit hooks (recommended) The pre-commit hooks for mlxtend will check your code via flake8 , black , and isort automatically before you make a git commit . You can read more about pre-commit hooks here . [ ] Install the pre-commit package via pip install pre-commit . [ ] In the mlxtend folder, run pre-commit install (you only have to do it once).","title":"2) Checking code style:"},{"location":"CONTRIBUTING/#3-submitting-your-code","text":"[ ] Push the topic branch to the server and create a pull request. [ ] Check the automated tests passed. [ ] The automatic PEP8 / black integrations may prompt you to modify the code stylistically. It would be nice if you could apply the suggested changes.","title":"3) Submitting your code"},{"location":"CONTRIBUTING/#tips-for-contributors","text":"","title":"Tips for Contributors"},{"location":"CONTRIBUTING/#getting-started-creating-a-new-issue-and-forking-the-repository","text":"If you don't have a GitHub account, yet, please create one to contribute to this project. Please submit a ticket for your issue to discuss the fix or new feature before too much time and effort is spent for the implementation. Fork the mlxtend repository from the GitHub web interface. Clone the mlxtend repository to your local machine by executing git clone https://github.com/<your_username>/mlxtend.git","title":"Getting Started - Creating a New Issue and Forking the Repository"},{"location":"CONTRIBUTING/#syncing-an-existing-fork","text":"If you already forked mlxtend earlier, you can bring you \"Fork\" up to date with the master branch as follows:","title":"Syncing an Existing Fork"},{"location":"CONTRIBUTING/#1-configuring-a-remote-that-points-to-the-upstream-repository-on-github","text":"List the current configured remote repository of your fork by executing $ git remote -v If you see something like origin https://github.com/<your username>/mlxtend.git (fetch) origin https://github.com/<your username>/mlxtend.git (push) you need to specify a new remote upstream repository via $ git remote add upstream https://github.com/rasbt/mlxtend.git Now, verify the new upstream repository you've specified for your fork by executing $ git remote -v You should see following output if everything is configured correctly: origin https://github.com/<your username>/mlxtend.git (fetch) origin https://github.com/<your username>/mlxtend.git (push) upstream https://github.com/rasbt/mlxtend.git (fetch) upstream https://github.com/rasbt/mlxtend.git (push)","title":"1. Configuring a remote that points to the upstream repository on GitHub"},{"location":"CONTRIBUTING/#2-syncing-your-fork","text":"First, fetch the updates of the original project's master branch by executing: $ git fetch upstream You should see the following output remote: Counting objects: xx, done. remote: Compressing objects: 100% (xx/xx), done. remote: Total xx (delta xx), reused xx (delta x) Unpacking objects: 100% (xx/xx), done. From https://github.com/rasbt/mlxtend * [new branch] master -> upstream/master This means that the commits to the rasbt/mlxtend master branch are now stored in the local branch upstream/master . If you are not already on your local project's master branch, execute $ git checkout master Finally, merge the changes in upstream/master to your local master branch by executing $ git merge upstream/master which will give you an output that looks similar to Updating xxx...xxx Fast-forward SOME FILE1 | 12 +++++++ SOME FILE2 | 10 +++++++ 2 files changed, 22 insertions(+),","title":"2. Syncing your Fork"},{"location":"CONTRIBUTING/#the-main-workflow-making-changes-in-a-new-topic-branch","text":"Listed below are the 9 typical steps of a contribution.","title":"*The Main Workflow - Making Changes in a New Topic Branch"},{"location":"CONTRIBUTING/#1-discussing-the-feature-or-modification","text":"Before you start coding, please discuss the new feature, bugfix, or other modification to the project on the project's issue tracker . Before you open a \"new issue,\" please do a quick search to see if a similar issue has been submitted already.","title":"1. Discussing the Feature or Modification"},{"location":"CONTRIBUTING/#2-creating-a-new-feature-branch","text":"Please avoid working directly on the master branch but create a new feature branch: $ git branch <new_feature> Switch to the new feature branch by executing $ git checkout <new_feature>","title":"2. Creating a new feature branch"},{"location":"CONTRIBUTING/#3-developing-the-new-feature-bug-fix","text":"Now it's time to modify existing code or to contribute new code to the project.","title":"3. Developing the new feature / bug fix"},{"location":"CONTRIBUTING/#4-testing-your-code","text":"Add the respective unit tests and check if they pass: $ PYTHONPATH='.' pytest ./mlxtend ---with-coverage","title":"4. Testing your code"},{"location":"CONTRIBUTING/#5-documenting-changes","text":"Please add an entry to the mlxtend/docs/sources/changelog.md file. If it is a new feature, it would also be nice if you could update the documentation in appropriate location in mlxtend/sources .","title":"5. Documenting changes"},{"location":"CONTRIBUTING/#6-committing-changes","text":"When you are ready to commit the changes, please provide a meaningful commit message: $ git add <modifies_files> # or `git add .` $ git commit -m '<meaningful commit message>'","title":"6. Committing changes"},{"location":"CONTRIBUTING/#7-optional-squashing-commits","text":"If you made multiple smaller commits, it would be nice if you could group them into a larger, summarizing commit. First, list your recent commit via Note Due to the improved GitHub UI, this is no longer necessary/encouraged. $ git log which will list the commits from newest to oldest in the following format by default: commit 046e3af8a9127df8eac879454f029937c8a31c41 Author: rasbt <mail@sebastianraschka.com> Date: Tue Nov 24 03:46:37 2015 -0500 fixed setup.py commit c3c00f6ba0e8f48bbe1c9081b8ae3817e57ecc5c Author: rasbt <mail@sebastianraschka.com> Date: Tue Nov 24 03:04:39 2015 -0500 documented feature x commit d87934fe8726c46f0b166d6290a3bf38915d6e75 Author: rasbt <mail@sebastianraschka.com> Date: Tue Nov 24 02:44:45 2015 -0500 added support for feature x Assuming that it would make sense to group these 3 commits into one, we can execute $ git rebase -i HEAD~3 which will bring our default git editor with the following contents: pick d87934f added support for feature x pick c3c00f6 documented feature x pick 046e3af fixed setup.py Since c3c00f6 and 046e3af are related to the original commit of feature x , let's keep the d87934f and squash the 2 following commits into this initial one by changes the lines to pick d87934f added support for feature x squash c3c00f6 documented feature x squash 046e3af fixed setup.py Now, save the changes in your editor. Now, quitting the editor will apply the rebase changes, and the editor will open a second time, prompting you to enter a new commit message. In this case, we could enter support for feature x to summarize the contributions.","title":"7. Optional: squashing commits"},{"location":"CONTRIBUTING/#8-uploading-changes","text":"Push your changes to a topic branch to the git server by executing: $ git push origin <feature_branch>","title":"8. Uploading changes"},{"location":"CONTRIBUTING/#9-submitting-a-pull-request","text":"Go to your GitHub repository online, select the new feature branch, and submit a new pull request:","title":"9. Submitting a pull request"},{"location":"CONTRIBUTING/#notes-for-developers","text":"","title":"Notes for Developers"},{"location":"CONTRIBUTING/#building-the-documentation","text":"The documentation is built via MkDocs ; to ensure that the documentation is rendered correctly, you can view the documentation locally by executing mkdocs serve from the mlxtend/docs directory. For example, ~/github/mlxtend/docs$ mkdocs serve","title":"Building the documentation"},{"location":"CONTRIBUTING/#1-building-the-api-documentation","text":"To build the API documentation, navigate to mlxtend/docs and execute the make_api.py file from this directory via ~/github/mlxtend/docs$ python make_api.py This should place the API documentation into the correct directories into the two directories: mlxtend/docs/sources/api_modules mlxtend/docs/sources/api_subpackes","title":"1. Building the API documentation"},{"location":"CONTRIBUTING/#2-editing-the-user-guide","text":"The documents containing code examples for the \"User Guide\" are generated from IPython Notebook files. In order to convert a IPython notebook file to markdown after editing, please follow the following steps: Modify or edit the existing notebook. Execute all cells in the current notebook and make sure that no errors occur. Convert the notebook to markdown using the ipynb2markdown.py converter ~/github/mlxtend/docs$ python ipynb2markdown.py --ipynb ./sources/user_guide/subpackage/notebookname.ipynb Note If you are adding a new document, please also include it in the pages section in the mlxtend/docs/mkdocs.yml file.","title":"2. Editing the User Guide"},{"location":"CONTRIBUTING/#3-building-static-html-files-of-the-documentation","text":"First, please check the documenation via localhost (https://127.0.0.1:8000/): ~/github/mlxtend/docs$ mkdocs serve Next, build the static HTML files of the mlxtend documentation via ~/github/mlxtend/docs$ mkdocs build --clean To deploy the documentation, execute ~/github/mlxtend/docs$ mkdocs gh-deploy --clean","title":"3. Building static HTML files of the documentation"},{"location":"CONTRIBUTING/#4-generate-a-pdf-of-the-documentation","text":"To generate a PDF version of the documentation, simply cd into the mlxtend/docs directory and execute: python md2pdf.py","title":"4. Generate a PDF of the documentation"},{"location":"CONTRIBUTING/#uploading-a-new-version-to-pypi","text":"","title":"Uploading a new version to PyPI"},{"location":"CONTRIBUTING/#1-creating-a-new-testing-environment","text":"Assuming we are using conda , create a new python environment via $ conda create -n 'mlxtend-testing' python=3 numpy scipy pandas Next, activate the environment by executing $ source activate mlxtend-testing","title":"1. Creating a new testing environment"},{"location":"CONTRIBUTING/#2-installing-the-package-from-local-files","text":"Test the installation by executing the following from within the mlxtend main directory: $ pip install . -e the --record files.txt flag will create a files.txt file listing the locations where these files will be installed. Try to import the package to see if it works, for example, by executing $ python -c 'import mlxtend; print(mlxtend.__file__)' If everything seems to be fine, remove the installation via $ cat files.txt | xargs rm -rf ; rm files.txt Next, test if pip is able to install the packages. First, navigate to a different directory, and from there, install the package: $ pip uninstall $ pip install mlxtend and uninstall it again $ pip uninstall mlxtend","title":"2. Installing the package from local files"},{"location":"CONTRIBUTING/#3-deploying-the-package","text":"Consider deploying the package to the PyPI test server first. The setup instructions can be found here . First , install Twine if you don't have it already installed. E.g., use the following to install all recommended packages: $ python -m pip install twine build Second , create the distribution. This by default creates an sdist and wheel in the ./dist directory. $ python -m build Install the wheel and sdist to make sure they work. The distributions file names will change with each version. python -m pip install ./dist/mlxtend-0.23.0.dev0.tar.gz --force-reinstall python -m pip install ./dist/mlxtend-0.23.0.dev0-py3-none-any.whl --force-reinstall python -m pip uninstall mlxtend Third , upload the packages to the test server: $ twine upload --repository-url https://upload.pypi.org/legacy dist/ Then, install it and see if it works: $ pip install -i https://testpypi.python.org/pypi mlxtend Next, uninstall it again as follows: $ pip uninstall mlxtend Fourth , after this dry-run succeeded, repeat this process using the \"real\" PyPI: $ python -m twine upload dist/*","title":"3. Deploying the package"},{"location":"CONTRIBUTING/#4-removing-the-virtual-environment","text":"Finally, to cleanup our local drive, remove the virtual testing environment via $ conda remove --name 'mlxtend-testing' --all Note : if you get an error like HTTPError: 403 Forbidden from https://upload.pypi.org/legacy/ make sure you have an up to date version of twine installed (helped me to uninstall in conda and install via pip.)","title":"4. Removing the virtual environment"},{"location":"CONTRIBUTING/#5-updating-the-conda-forge-recipe","text":"Once a new version of mlxtend has been uploaded to PyPI, update the conda-forge build recipe at https://github.com/conda-forge/mlxtend-feedstock by changing the version number in the recipe/meta.yaml file appropriately.","title":"5. Updating the conda-forge recipe"},{"location":"Code-of-Conduct/","text":"Contributor Covenant Code of Conduct Our Pledge In the interest of fostering an open and welcoming environment, we as contributors and maintainers pledge to make participation in our project and our community a harassment-free experience for everyone, regardless of age, body size, disability, ethnicity, sex characteristics, gender identity, and expression, level of experience, education, socio-economic status, nationality, personal appearance, race, religion, or sexual identity and orientation. Our Standards Examples of behavior that contributes to creating a positive environment include: Using welcoming and inclusive language; Being respectful of differing viewpoints and experiences; Gracefully accepting constructive criticism; Focusing on what is best for the community; Showing empathy towards other community members. Examples of unacceptable behavior by participants include: The use of sexualized language or imagery and unwelcome sexual attention or advances; Trolling, gaslighting, insulting/derogatory comments, and personal or political attacks; Public or private harassment; Publishing others' private information, such as a physical or electronic address, without explicit permission; Other conduct which could reasonably be considered inappropriate in a professional setting. Our Responsibilities Project maintainers are responsible for clarifying the standards of acceptable behavior and are expected to take appropriate and fair corrective action in response to any instances of unacceptable behavior. Project maintainers have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct, or to ban temporarily or permanently any contributor for other behaviors that they deem inappropriate, threatening, offensive, or harmful. Scope This Code of Conduct applies both within the project and public spaces when an individual represents the project or its community. Examples of representing a project or community include using an official project e-mail address, posting via an official social media account or acting as an appointed representative at an online or offline event. Representation of a project may be further defined and clarified by project maintainers. Enforcement Instances of abusive, harassing, or other unacceptable behavior may be reported by contacting the project team at mail@sebastianraschka.com. All complaints will be reviewed and investigated and will result in a response that is deemed necessary and appropriate to the circumstances. The project team is obligated to maintain confidentiality concerning the reporter of an incident. Further details of specific enforcement policies may be posted separately. Project maintainers who do not follow or enforce the Code of Conduct in good faith may face temporary or permanent repercussions as determined by other members of the project's leadership. Attribution This Code of Conduct is adapted from the Contributor Covenant , version 1.4, available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html For answers to common questions about this code of conduct, see https://www.contributor-covenant.org/faq","title":"Code of Conduct"},{"location":"Code-of-Conduct/#contributor-covenant-code-of-conduct","text":"","title":"Contributor Covenant Code of Conduct"},{"location":"Code-of-Conduct/#our-pledge","text":"In the interest of fostering an open and welcoming environment, we as contributors and maintainers pledge to make participation in our project and our community a harassment-free experience for everyone, regardless of age, body size, disability, ethnicity, sex characteristics, gender identity, and expression, level of experience, education, socio-economic status, nationality, personal appearance, race, religion, or sexual identity and orientation.","title":"Our Pledge"},{"location":"Code-of-Conduct/#our-standards","text":"Examples of behavior that contributes to creating a positive environment include: Using welcoming and inclusive language; Being respectful of differing viewpoints and experiences; Gracefully accepting constructive criticism; Focusing on what is best for the community; Showing empathy towards other community members. Examples of unacceptable behavior by participants include: The use of sexualized language or imagery and unwelcome sexual attention or advances; Trolling, gaslighting, insulting/derogatory comments, and personal or political attacks; Public or private harassment; Publishing others' private information, such as a physical or electronic address, without explicit permission; Other conduct which could reasonably be considered inappropriate in a professional setting.","title":"Our Standards"},{"location":"Code-of-Conduct/#our-responsibilities","text":"Project maintainers are responsible for clarifying the standards of acceptable behavior and are expected to take appropriate and fair corrective action in response to any instances of unacceptable behavior. Project maintainers have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct, or to ban temporarily or permanently any contributor for other behaviors that they deem inappropriate, threatening, offensive, or harmful.","title":"Our Responsibilities"},{"location":"Code-of-Conduct/#scope","text":"This Code of Conduct applies both within the project and public spaces when an individual represents the project or its community. Examples of representing a project or community include using an official project e-mail address, posting via an official social media account or acting as an appointed representative at an online or offline event. Representation of a project may be further defined and clarified by project maintainers.","title":"Scope"},{"location":"Code-of-Conduct/#enforcement","text":"Instances of abusive, harassing, or other unacceptable behavior may be reported by contacting the project team at mail@sebastianraschka.com. All complaints will be reviewed and investigated and will result in a response that is deemed necessary and appropriate to the circumstances. The project team is obligated to maintain confidentiality concerning the reporter of an incident. Further details of specific enforcement policies may be posted separately. Project maintainers who do not follow or enforce the Code of Conduct in good faith may face temporary or permanent repercussions as determined by other members of the project's leadership.","title":"Enforcement"},{"location":"Code-of-Conduct/#attribution","text":"This Code of Conduct is adapted from the Contributor Covenant , version 1.4, available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html For answers to common questions about this code of conduct, see https://www.contributor-covenant.org/faq","title":"Attribution"},{"location":"USER-GUIDE-INDEX/","text":"User Guide Index classifier Adaline EnsembleVoteClassifier LogisticRegression MultiLayerPerceptron OneRClassifier Perceptron SoftmaxRegression StackingClassifier StackingCVClassifier cluster Kmeans data autompg_data boston_housing_data iris_data loadlocal_mnist make_multiplexer_dataset mnist_data three_blobs_data wine_data evaluate accuracy_score bias_variance_decomp bootstrap bootstrap_point632_score BootstrapOutOfBag cochrans_q combined_ftest_5x2cv confusion_matrix create_counterfactual feature_importance_permutation ftest lift_score mcnemar_table mcnemar_tables mcnemar paired_ttest_5x2cv paired_ttest_kfold_cv paired_ttest_resampled permutation_test PredefinedHoldoutSplit proportion_difference RandomHoldoutSplit scoring feature_extraction LinearDiscriminantAnalysis PrincipalComponentAnalysis RBFKernelPCA feature_selection ColumnSelector ExhaustiveFeatureSelector SequentialFeatureSelector file_io find_filegroups find_files frequent_patterns apriori association_rules fpgrowth fpmax math num_combinations num_permutations plotting category_scatter checkerboard_plot ecdf enrichment_plot heatmap plot_confusion_matrix plot_pca_correlation_graph plot_decision_regions plot_learning_curves plot_linear_regression plot_sequential_feature_selection scatterplotmatrix stacked_barplot preprocessing CopyTransformer DenseTransformer MeanCenterer minmax_scaling one-hot_encoding shuffle_arrays_unison standardize TransactionEncoder regressor LinearRegression StackingCVRegressor StackingRegressor text generalize_names generalize_names_duplcheck tokenizer utils Counter","title":"User Guide Index"},{"location":"USER-GUIDE-INDEX/#user-guide-index","text":"","title":"User Guide Index"},{"location":"USER-GUIDE-INDEX/#classifier","text":"Adaline EnsembleVoteClassifier LogisticRegression MultiLayerPerceptron OneRClassifier Perceptron SoftmaxRegression StackingClassifier StackingCVClassifier","title":"classifier"},{"location":"USER-GUIDE-INDEX/#cluster","text":"Kmeans","title":"cluster"},{"location":"USER-GUIDE-INDEX/#data","text":"autompg_data boston_housing_data iris_data loadlocal_mnist make_multiplexer_dataset mnist_data three_blobs_data wine_data","title":"data"},{"location":"USER-GUIDE-INDEX/#evaluate","text":"accuracy_score bias_variance_decomp bootstrap bootstrap_point632_score BootstrapOutOfBag cochrans_q combined_ftest_5x2cv confusion_matrix create_counterfactual feature_importance_permutation ftest lift_score mcnemar_table mcnemar_tables mcnemar paired_ttest_5x2cv paired_ttest_kfold_cv paired_ttest_resampled permutation_test PredefinedHoldoutSplit proportion_difference RandomHoldoutSplit scoring","title":"evaluate"},{"location":"USER-GUIDE-INDEX/#feature_extraction","text":"LinearDiscriminantAnalysis PrincipalComponentAnalysis RBFKernelPCA","title":"feature_extraction"},{"location":"USER-GUIDE-INDEX/#feature_selection","text":"ColumnSelector ExhaustiveFeatureSelector SequentialFeatureSelector","title":"feature_selection"},{"location":"USER-GUIDE-INDEX/#file_io","text":"find_filegroups find_files","title":"file_io"},{"location":"USER-GUIDE-INDEX/#frequent_patterns","text":"apriori association_rules fpgrowth fpmax","title":"frequent_patterns"},{"location":"USER-GUIDE-INDEX/#math","text":"num_combinations num_permutations","title":"math"},{"location":"USER-GUIDE-INDEX/#plotting","text":"category_scatter checkerboard_plot ecdf enrichment_plot heatmap plot_confusion_matrix plot_pca_correlation_graph plot_decision_regions plot_learning_curves plot_linear_regression plot_sequential_feature_selection scatterplotmatrix stacked_barplot","title":"plotting"},{"location":"USER-GUIDE-INDEX/#preprocessing","text":"CopyTransformer DenseTransformer MeanCenterer minmax_scaling one-hot_encoding shuffle_arrays_unison standardize TransactionEncoder","title":"preprocessing"},{"location":"USER-GUIDE-INDEX/#regressor","text":"LinearRegression StackingCVRegressor StackingRegressor","title":"regressor"},{"location":"USER-GUIDE-INDEX/#text","text":"generalize_names generalize_names_duplcheck tokenizer","title":"text"},{"location":"USER-GUIDE-INDEX/#utils","text":"Counter","title":"utils"},{"location":"USER_GUIDE_INDEX/","text":"User Guide Index classifier Adaline EnsembleVoteClassifier LogisticRegression MultiLayerPerceptron OneRClassifier Perceptron SoftmaxRegression StackingClassifier StackingCVClassifier cluster Kmeans data autompg_data boston_housing_data iris_data loadlocal_mnist make_multiplexer_dataset mnist_data three_blobs_data wine_data evaluate accuracy_score bias_variance_decomp bootstrap bootstrap_point632_score BootstrapOutOfBag cochrans_q combined_ftest_5x2cv confusion_matrix create_counterfactual feature_importance_permutation ftest GroupTimeSeriesSplit lift_score mcnemar_table mcnemar_tables mcnemar paired_ttest_5x2cv paired_ttest_kfold_cv paired_ttest_resampled permutation_test PredefinedHoldoutSplit proportion_difference RandomHoldoutSplit scoring feature_extraction LinearDiscriminantAnalysis PrincipalComponentAnalysis RBFKernelPCA feature_selection ColumnSelector ExhaustiveFeatureSelector SequentialFeatureSelector file_io find_filegroups find_files frequent_patterns apriori association_rules fpgrowth fpmax math num_combinations num_permutations plotting category_scatter checkerboard_plot ecdf enrichment_plot heatmap plot_confusion_matrix plot_pca_correlation_graph plot_decision_regions plot_learning_curves plot_linear_regression plot_sequential_feature_selection scatterplotmatrix scatter_hist stacked_barplot preprocessing CopyTransformer DenseTransformer MeanCenterer minmax_scaling one-hot_encoding shuffle_arrays_unison standardize TransactionEncoder regressor LinearRegression StackingCVRegressor StackingRegressor text generalize_names generalize_names_duplcheck tokenizer utils Counter","title":"User Guide Index"},{"location":"USER_GUIDE_INDEX/#user-guide-index","text":"","title":"User Guide Index"},{"location":"USER_GUIDE_INDEX/#classifier","text":"Adaline EnsembleVoteClassifier LogisticRegression MultiLayerPerceptron OneRClassifier Perceptron SoftmaxRegression StackingClassifier StackingCVClassifier","title":"classifier"},{"location":"USER_GUIDE_INDEX/#cluster","text":"Kmeans","title":"cluster"},{"location":"USER_GUIDE_INDEX/#data","text":"autompg_data boston_housing_data iris_data loadlocal_mnist make_multiplexer_dataset mnist_data three_blobs_data wine_data","title":"data"},{"location":"USER_GUIDE_INDEX/#evaluate","text":"accuracy_score bias_variance_decomp bootstrap bootstrap_point632_score BootstrapOutOfBag cochrans_q combined_ftest_5x2cv confusion_matrix create_counterfactual feature_importance_permutation ftest GroupTimeSeriesSplit lift_score mcnemar_table mcnemar_tables mcnemar paired_ttest_5x2cv paired_ttest_kfold_cv paired_ttest_resampled permutation_test PredefinedHoldoutSplit proportion_difference RandomHoldoutSplit scoring","title":"evaluate"},{"location":"USER_GUIDE_INDEX/#feature_extraction","text":"LinearDiscriminantAnalysis PrincipalComponentAnalysis RBFKernelPCA","title":"feature_extraction"},{"location":"USER_GUIDE_INDEX/#feature_selection","text":"ColumnSelector ExhaustiveFeatureSelector SequentialFeatureSelector","title":"feature_selection"},{"location":"USER_GUIDE_INDEX/#file_io","text":"find_filegroups find_files","title":"file_io"},{"location":"USER_GUIDE_INDEX/#frequent_patterns","text":"apriori association_rules fpgrowth fpmax","title":"frequent_patterns"},{"location":"USER_GUIDE_INDEX/#math","text":"num_combinations num_permutations","title":"math"},{"location":"USER_GUIDE_INDEX/#plotting","text":"category_scatter checkerboard_plot ecdf enrichment_plot heatmap plot_confusion_matrix plot_pca_correlation_graph plot_decision_regions plot_learning_curves plot_linear_regression plot_sequential_feature_selection scatterplotmatrix scatter_hist stacked_barplot","title":"plotting"},{"location":"USER_GUIDE_INDEX/#preprocessing","text":"CopyTransformer DenseTransformer MeanCenterer minmax_scaling one-hot_encoding shuffle_arrays_unison standardize TransactionEncoder","title":"preprocessing"},{"location":"USER_GUIDE_INDEX/#regressor","text":"LinearRegression StackingCVRegressor StackingRegressor","title":"regressor"},{"location":"USER_GUIDE_INDEX/#text","text":"generalize_names generalize_names_duplcheck tokenizer","title":"text"},{"location":"USER_GUIDE_INDEX/#utils","text":"Counter","title":"utils"},{"location":"cite/","text":"Citing mlxtend If you use mlxtend as part of your workflow in a scientific publication, please consider citing the mlxtend repository with the following DOI: Raschka, Sebastian (2018) MLxtend: Providing machine learning and data science utilities and extensions to Python's scientific computing stack . J Open Source Softw 3(24). @article{raschkas_2018_mlxtend, author = {Sebastian Raschka}, title = {MLxtend: Providing machine learning and data science utilities and extensions to Python\u2019s scientific computing stack}, journal = {The Journal of Open Source Software}, volume = {3}, number = {24}, month = apr, year = 2018, publisher = {The Open Journal}, doi = {10.21105/joss.00638}, url = {https://joss.theoj.org/papers/10.21105/joss.00638} }","title":"Citing Mlxtend"},{"location":"cite/#citing-mlxtend","text":"If you use mlxtend as part of your workflow in a scientific publication, please consider citing the mlxtend repository with the following DOI: Raschka, Sebastian (2018) MLxtend: Providing machine learning and data science utilities and extensions to Python's scientific computing stack . J Open Source Softw 3(24). @article{raschkas_2018_mlxtend, author = {Sebastian Raschka}, title = {MLxtend: Providing machine learning and data science utilities and extensions to Python\u2019s scientific computing stack}, journal = {The Journal of Open Source Software}, volume = {3}, number = {24}, month = apr, year = 2018, publisher = {The Open Journal}, doi = {10.21105/joss.00638}, url = {https://joss.theoj.org/papers/10.21105/joss.00638} }","title":"Citing mlxtend"},{"location":"contributors/","text":"Contributors For the current list of contributors to mlxtend, please see the GitHub contributor page at https://github.com/rasbt/mlxtend/graphs/contributors .","title":"Contributors"},{"location":"contributors/#contributors","text":"For the current list of contributors to mlxtend, please see the GitHub contributor page at https://github.com/rasbt/mlxtend/graphs/contributors .","title":"Contributors"},{"location":"discuss/","text":"Discuss Any questions or comments about mlxtend? Join the mlxtend mailing list on Google Groups!","title":"Discuss"},{"location":"discuss/#discuss","text":"Any questions or comments about mlxtend? Join the mlxtend mailing list on Google Groups!","title":"Discuss"},{"location":"installation/","text":"Installing mlxtend PyPI To install mlxtend, just execute pip install mlxtend Alternatively, you download the package manually from the Python Package Index https://pypi.python.org/pypi/mlxtend , unzip it, navigate into the package, and use the following command from inside the mlxtend folder: pip install . Upgrading via pip To upgrade an existing version of mlxtend from PyPI, execute pip install mlxtend --upgrade --no-deps Please note that the dependencies (NumPy and SciPy) will also be upgraded if you omit the --no-deps flag; use the --no-deps (\"no dependencies\") flag if you don't want this. Installing mlxtend from the source distribution In rare cases, users reported problems on certain systems with the default pip installation command, which installs mlxtend from the binary distribution (\"wheels\") on PyPI. If you should encounter similar problems, you could try to install mlxtend from the source distribution instead via pip install --no-binary :all: mlxtend Also, I would appreciate it if you could report any issues that occur when using pip install mlxtend in hope that we can fix these in future releases. Conda The mlxtend package is also available through conda forge . To install mlxtend using conda, use the following command: conda install mlxtend --channel conda-forge or simply conda install mlxtend if you added conda-forge to your channels ( conda config --add channels conda-forge ). Dev Version The mlxtend version on PyPI may always one step behind; you can install the latest development version from the GitHub repository by executing pip install git+git://github.com/rasbt/mlxtend.git Or, you can fork the GitHub repository from https://github.com/rasbt/mlxtend and install mlxtend from your local drive via pip install .","title":"Installation"},{"location":"installation/#installing-mlxtend","text":"","title":"Installing mlxtend"},{"location":"installation/#pypi","text":"To install mlxtend, just execute pip install mlxtend Alternatively, you download the package manually from the Python Package Index https://pypi.python.org/pypi/mlxtend , unzip it, navigate into the package, and use the following command from inside the mlxtend folder: pip install .","title":"PyPI"},{"location":"installation/#upgrading-via-pip","text":"To upgrade an existing version of mlxtend from PyPI, execute pip install mlxtend --upgrade --no-deps Please note that the dependencies (NumPy and SciPy) will also be upgraded if you omit the --no-deps flag; use the --no-deps (\"no dependencies\") flag if you don't want this.","title":"Upgrading via pip"},{"location":"installation/#installing-mlxtend-from-the-source-distribution","text":"In rare cases, users reported problems on certain systems with the default pip installation command, which installs mlxtend from the binary distribution (\"wheels\") on PyPI. If you should encounter similar problems, you could try to install mlxtend from the source distribution instead via pip install --no-binary :all: mlxtend Also, I would appreciate it if you could report any issues that occur when using pip install mlxtend in hope that we can fix these in future releases.","title":"Installing mlxtend from the source distribution"},{"location":"installation/#conda","text":"The mlxtend package is also available through conda forge . To install mlxtend using conda, use the following command: conda install mlxtend --channel conda-forge or simply conda install mlxtend if you added conda-forge to your channels ( conda config --add channels conda-forge ).","title":"Conda"},{"location":"installation/#dev-version","text":"The mlxtend version on PyPI may always one step behind; you can install the latest development version from the GitHub repository by executing pip install git+git://github.com/rasbt/mlxtend.git Or, you can fork the GitHub repository from https://github.com/rasbt/mlxtend and install mlxtend from your local drive via pip install .","title":"Dev Version"},{"location":"license/","text":"This project is released under a permissive new BSD open source license and commercially usable. There is no warranty; not even for merchantability or fitness for a particular purpose. In addition, you may use, copy, modify, and redistribute all artistic creative works (figures and images) included in this distribution under the directory according to the terms and conditions of the Creative Commons Attribution 4.0 International License. (Computer-generated graphics such as the plots produced by matplotlib fall under the BSD license mentioned above). new BSD License New BSD License Copyright (c) 2014-2024, Sebastian Raschka. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. Neither the name of mlxtend nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \"AS IS\" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. Creative Commons Attribution 4.0 International License mlxtend documentation figures are licensed under a Creative Commons Attribution 4.0 International License. https://creativecommons.org/licenses/by-sa/4.0/ . You are free to: Share \u2014 copy and redistribute the material in any medium or format Adapt \u2014 remix, transform, and build upon the material for any purpose, even commercially. The licensor cannot revoke these freedoms as long as you follow the license terms. Under the following terms: Attribution \u2014 You must give appropriate credit, provide a link to the license, and indicate if changes were made. You may do so in any reasonable manner, but not in any way that suggests the licensor endorses you or your use. No additional restrictions \u2014 You may not apply legal terms or technological measures that legally restrict others from doing anything the license permits.","title":"License"},{"location":"license/#new-bsd-license","text":"New BSD License Copyright (c) 2014-2024, Sebastian Raschka. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. Neither the name of mlxtend nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \"AS IS\" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.","title":"new BSD License"},{"location":"license/#creative-commons-attribution-40-international-license","text":"mlxtend documentation figures are licensed under a Creative Commons Attribution 4.0 International License. https://creativecommons.org/licenses/by-sa/4.0/ .","title":"Creative Commons Attribution 4.0 International License"},{"location":"license/#you-are-free-to","text":"Share \u2014 copy and redistribute the material in any medium or format Adapt \u2014 remix, transform, and build upon the material for any purpose, even commercially. The licensor cannot revoke these freedoms as long as you follow the license terms.","title":"You are free to:"},{"location":"license/#under-the-following-terms","text":"Attribution \u2014 You must give appropriate credit, provide a link to the license, and indicate if changes were made. You may do so in any reasonable manner, but not in any way that suggests the licensor endorses you or your use. No additional restrictions \u2014 You may not apply legal terms or technological measures that legally restrict others from doing anything the license permits.","title":"Under the following terms:"},{"location":"api_modules/mlxtend.classifier/Adaline/","text":"Adaline Adaline(eta=0.01, epochs=50, minibatches=None, random_seed=None, print_progress=0) ADAptive LInear NEuron classifier. Note that this implementation of Adaline expects binary class labels in {0, 1}. Parameters eta : float (default: 0.01) solver rate (between 0.0 and 1.0) epochs : int (default: 50) Passes over the training dataset. Prior to each epoch, the dataset is shuffled if minibatches > 1 to prevent cycles in stochastic gradient descent. minibatches : int (default: None) The number of minibatches for gradient-based optimization. If None: Normal Equations (closed-form solution) If 1: Gradient Descent learning If len(y): Stochastic Gradient Descent (SGD) online learning If 1 < minibatches < len(y): SGD Minibatch learning random_seed : int (default: None) Set random state for shuffling and initializing the weights. print_progress : int (default: 0) Prints progress in fitting to stderr if not solver='normal equation' 0: No output 1: Epochs elapsed and cost 2: 1 plus time elapsed 3: 2 plus estimated time until completion Attributes w_ : 2d-array, shape={n_features, 1} Model weights after fitting. b_ : 1d-array, shape={1,} Bias unit after fitting. cost_ : list Sum of squared errors after each epoch. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/classifier/Adaline/ Methods fit(X, y, init_params=True) Learn model from training data. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values. init_params : bool (default: True) Re-initializes model parameters prior to fitting. Set False to continue training with weights from a previous model fitting. Returns self : object get_params(deep=True) Get parameters for this estimator. Parameters deep : boolean, optional If True, will return the parameters for this estimator and contained subobjects that are estimators. Returns params : mapping of string to any Parameter names mapped to their values.' adapted from https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/base.py Author: Gael Varoquaux gael.varoquaux@normalesup.org License: BSD 3 clause predict(X) Predict targets from X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns target_values : array-like, shape = [n_samples] Predicted target values. score(X, y) Compute the prediction accuracy Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values (true class labels). Returns acc : float The prediction accuracy as a float between 0.0 and 1.0 (perfect score). set_params( params) Set the parameters of this estimator. The method works on simple estimators as well as on nested objects (such as pipelines). The latter have parameters of the form <component>__<parameter> so that it's possible to update each component of a nested object. Returns self adapted from https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/base.py Author: Gael Varoquaux <gael.varoquaux@normalesup.org> License: BSD 3 clause","title":"Adaline"},{"location":"api_modules/mlxtend.classifier/Adaline/#adaline","text":"Adaline(eta=0.01, epochs=50, minibatches=None, random_seed=None, print_progress=0) ADAptive LInear NEuron classifier. Note that this implementation of Adaline expects binary class labels in {0, 1}. Parameters eta : float (default: 0.01) solver rate (between 0.0 and 1.0) epochs : int (default: 50) Passes over the training dataset. Prior to each epoch, the dataset is shuffled if minibatches > 1 to prevent cycles in stochastic gradient descent. minibatches : int (default: None) The number of minibatches for gradient-based optimization. If None: Normal Equations (closed-form solution) If 1: Gradient Descent learning If len(y): Stochastic Gradient Descent (SGD) online learning If 1 < minibatches < len(y): SGD Minibatch learning random_seed : int (default: None) Set random state for shuffling and initializing the weights. print_progress : int (default: 0) Prints progress in fitting to stderr if not solver='normal equation' 0: No output 1: Epochs elapsed and cost 2: 1 plus time elapsed 3: 2 plus estimated time until completion Attributes w_ : 2d-array, shape={n_features, 1} Model weights after fitting. b_ : 1d-array, shape={1,} Bias unit after fitting. cost_ : list Sum of squared errors after each epoch. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/classifier/Adaline/","title":"Adaline"},{"location":"api_modules/mlxtend.classifier/Adaline/#methods","text":"fit(X, y, init_params=True) Learn model from training data. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values. init_params : bool (default: True) Re-initializes model parameters prior to fitting. Set False to continue training with weights from a previous model fitting. Returns self : object get_params(deep=True) Get parameters for this estimator. Parameters deep : boolean, optional If True, will return the parameters for this estimator and contained subobjects that are estimators. Returns params : mapping of string to any Parameter names mapped to their values.' adapted from https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/base.py Author: Gael Varoquaux gael.varoquaux@normalesup.org License: BSD 3 clause predict(X) Predict targets from X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns target_values : array-like, shape = [n_samples] Predicted target values. score(X, y) Compute the prediction accuracy Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values (true class labels). Returns acc : float The prediction accuracy as a float between 0.0 and 1.0 (perfect score). set_params( params) Set the parameters of this estimator. The method works on simple estimators as well as on nested objects (such as pipelines). The latter have parameters of the form <component>__<parameter> so that it's possible to update each component of a nested object. Returns self adapted from https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/base.py Author: Gael Varoquaux <gael.varoquaux@normalesup.org> License: BSD 3 clause","title":"Methods"},{"location":"api_modules/mlxtend.classifier/EnsembleVoteClassifier/","text":"EnsembleVoteClassifier EnsembleVoteClassifier(clfs, voting='hard', weights=None, verbose=0, use_clones=True, fit_base_estimators=True) Soft Voting/Majority Rule classifier for scikit-learn estimators. Parameters clfs : array-like, shape = [n_classifiers] A list of classifiers. Invoking the fit method on the VotingClassifier will fit clones of those original classifiers be stored in the class attribute if use_clones=True (default) and fit_base_estimators=True (default). voting : str, {'hard', 'soft'} (default='hard') If 'hard', uses predicted class labels for majority rule voting. Else if 'soft', predicts the class label based on the argmax of the sums of the predicted probalities, which is recommended for an ensemble of well-calibrated classifiers. weights : array-like, shape = [n_classifiers], optional (default= None ) Sequence of weights ( float or int ) to weight the occurances of predicted class labels ( hard voting) or class probabilities before averaging ( soft voting). Uses uniform weights if None . verbose : int, optional (default=0) Controls the verbosity of the building process. - verbose=0 (default): Prints nothing - verbose=1 : Prints the number & name of the clf being fitted - verbose=2 : Prints info about the parameters of the clf being fitted - verbose>2 : Changes verbose param of the underlying clf to self.verbose - 2 use_clones : bool (default: True) Clones the classifiers for stacking classification if True (default) or else uses the original ones, which will be refitted on the dataset upon calling the fit method. Hence, if use_clones=True, the original input classifiers will remain unmodified upon using the StackingClassifier's fit method. Setting use_clones=False is recommended if you are working with estimators that are supporting the scikit-learn fit/predict API interface but are not compatible to scikit-learn's clone function. fit_base_estimators : bool (default: True) Refits classifiers in clfs if True; uses references to the clfs , otherwise (assumes that the classifiers were already fit). Note: fit_base_estimators=False will enforce use_clones to be False, and is incompatible to most scikit-learn wrappers! For instance, if any form of cross-validation is performed this would require the re-fitting classifiers to training folds, which would raise a NotFitterError if fit_base_estimators=False. (New in mlxtend v0.6.) Attributes classes_ : array-like, shape = [n_predictions] clf : array-like, shape = [n_predictions] The input classifiers; may be overwritten if use_clones=False clf_ : array-like, shape = [n_predictions] Fitted input classifiers; clones if use_clones=True Examples ``` >>> import numpy as np >>> from sklearn.linear_model import LogisticRegression >>> from sklearn.naive_bayes import GaussianNB >>> from sklearn.ensemble import RandomForestClassifier >>> from mlxtend.sklearn import EnsembleVoteClassifier >>> clf1 = LogisticRegression(random_seed=1) >>> clf2 = RandomForestClassifier(random_seed=1) >>> clf3 = GaussianNB() >>> X = np.array([[-1, -1], [-2, -1], [-3, -2], [1, 1], [2, 1], [3, 2]]) >>> y = np.array([1, 1, 1, 2, 2, 2]) >>> eclf1 = EnsembleVoteClassifier(clfs=[clf1, clf2, clf3], ... voting='hard', verbose=1) >>> eclf1 = eclf1.fit(X, y) >>> print(eclf1.predict(X)) [1 1 1 2 2 2] >>> eclf2 = EnsembleVoteClassifier(clfs=[clf1, clf2, clf3], voting='soft') >>> eclf2 = eclf2.fit(X, y) >>> print(eclf2.predict(X)) [1 1 1 2 2 2] >>> eclf3 = EnsembleVoteClassifier(clfs=[clf1, clf2, clf3], ... voting='soft', weights=[2,1,1]) >>> eclf3 = eclf3.fit(X, y) >>> print(eclf3.predict(X)) [1 1 1 2 2 2] >>> For more usage examples, please see https://rasbt.github.io/mlxtend/user_guide/classifier/EnsembleVoteClassifier/ ``` Methods fit(X, y, sample_weight=None) Learn weight coefficients from training data for each classifier. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values. sample_weight : array-like, shape = [n_samples], optional Sample weights passed as sample_weights to each regressor in the regressors list as well as the meta_regressor. Raises error if some regressor does not support sample_weight in the fit() method. Returns self : object fit_transform(X, y=None, fit_params) Fit to data, then transform it. Fits transformer to `X` and `y` with optional parameters `fit_params` and returns a transformed version of `X`. Parameters X : array-like of shape (n_samples, n_features) Input samples. y : array-like of shape (n_samples,) or (n_samples, n_outputs), default=None Target values (None for unsupervised transformations). **fit_params : dict Additional fit parameters. Returns X_new : ndarray array of shape (n_samples, n_features_new) Transformed array. get_params(deep=True) Return estimator parameter names for GridSearch support. predict(X) Predict class labels for X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns maj : array-like, shape = [n_samples] Predicted class labels. predict_proba(X) Predict class probabilities for X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns avg : array-like, shape = [n_samples, n_classes] Weighted average probability for each class per sample. score(X, y, sample_weight=None) Return the mean accuracy on the given test data and labels. In multi-label classification, this is the subset accuracy which is a harsh metric since you require for each sample that each label set be correctly predicted. Parameters X : array-like of shape (n_samples, n_features) Test samples. y : array-like of shape (n_samples,) or (n_samples, n_outputs) True labels for X . sample_weight : array-like of shape (n_samples,), default=None Sample weights. Returns score : float Mean accuracy of self.predict(X) w.r.t. y . set_output( , transform=None)* Set output container. See :ref:`sphx_glr_auto_examples_miscellaneous_plot_set_output.py` for an example on how to use the API. Parameters transform : {\"default\", \"pandas\"}, default=None Configure output of transform and fit_transform . \"default\" : Default output format of a transformer \"pandas\" : DataFrame output None : Transform configuration is unchanged Returns self : estimator instance Estimator instance. set_params( params) Set the parameters of this estimator. The method works on simple estimators as well as on nested objects (such as :class:`~sklearn.pipeline.Pipeline`). The latter have parameters of the form ``<component>__<parameter>`` so that it's possible to update each component of a nested object. Parameters **params : dict Estimator parameters. Returns self : estimator instance Estimator instance. transform(X) Return class labels or probabilities for X for each estimator. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns If voting='soft'`` : array-like = [n_classifiers, n_samples, n_classes] Class probabilties calculated by each classifier. If voting='hard'`` : array-like = [n_classifiers, n_samples] Class labels predicted by each classifier.","title":"EnsembleVoteClassifier"},{"location":"api_modules/mlxtend.classifier/EnsembleVoteClassifier/#ensemblevoteclassifier","text":"EnsembleVoteClassifier(clfs, voting='hard', weights=None, verbose=0, use_clones=True, fit_base_estimators=True) Soft Voting/Majority Rule classifier for scikit-learn estimators. Parameters clfs : array-like, shape = [n_classifiers] A list of classifiers. Invoking the fit method on the VotingClassifier will fit clones of those original classifiers be stored in the class attribute if use_clones=True (default) and fit_base_estimators=True (default). voting : str, {'hard', 'soft'} (default='hard') If 'hard', uses predicted class labels for majority rule voting. Else if 'soft', predicts the class label based on the argmax of the sums of the predicted probalities, which is recommended for an ensemble of well-calibrated classifiers. weights : array-like, shape = [n_classifiers], optional (default= None ) Sequence of weights ( float or int ) to weight the occurances of predicted class labels ( hard voting) or class probabilities before averaging ( soft voting). Uses uniform weights if None . verbose : int, optional (default=0) Controls the verbosity of the building process. - verbose=0 (default): Prints nothing - verbose=1 : Prints the number & name of the clf being fitted - verbose=2 : Prints info about the parameters of the clf being fitted - verbose>2 : Changes verbose param of the underlying clf to self.verbose - 2 use_clones : bool (default: True) Clones the classifiers for stacking classification if True (default) or else uses the original ones, which will be refitted on the dataset upon calling the fit method. Hence, if use_clones=True, the original input classifiers will remain unmodified upon using the StackingClassifier's fit method. Setting use_clones=False is recommended if you are working with estimators that are supporting the scikit-learn fit/predict API interface but are not compatible to scikit-learn's clone function. fit_base_estimators : bool (default: True) Refits classifiers in clfs if True; uses references to the clfs , otherwise (assumes that the classifiers were already fit). Note: fit_base_estimators=False will enforce use_clones to be False, and is incompatible to most scikit-learn wrappers! For instance, if any form of cross-validation is performed this would require the re-fitting classifiers to training folds, which would raise a NotFitterError if fit_base_estimators=False. (New in mlxtend v0.6.) Attributes classes_ : array-like, shape = [n_predictions] clf : array-like, shape = [n_predictions] The input classifiers; may be overwritten if use_clones=False clf_ : array-like, shape = [n_predictions] Fitted input classifiers; clones if use_clones=True Examples ``` >>> import numpy as np >>> from sklearn.linear_model import LogisticRegression >>> from sklearn.naive_bayes import GaussianNB >>> from sklearn.ensemble import RandomForestClassifier >>> from mlxtend.sklearn import EnsembleVoteClassifier >>> clf1 = LogisticRegression(random_seed=1) >>> clf2 = RandomForestClassifier(random_seed=1) >>> clf3 = GaussianNB() >>> X = np.array([[-1, -1], [-2, -1], [-3, -2], [1, 1], [2, 1], [3, 2]]) >>> y = np.array([1, 1, 1, 2, 2, 2]) >>> eclf1 = EnsembleVoteClassifier(clfs=[clf1, clf2, clf3], ... voting='hard', verbose=1) >>> eclf1 = eclf1.fit(X, y) >>> print(eclf1.predict(X)) [1 1 1 2 2 2] >>> eclf2 = EnsembleVoteClassifier(clfs=[clf1, clf2, clf3], voting='soft') >>> eclf2 = eclf2.fit(X, y) >>> print(eclf2.predict(X)) [1 1 1 2 2 2] >>> eclf3 = EnsembleVoteClassifier(clfs=[clf1, clf2, clf3], ... voting='soft', weights=[2,1,1]) >>> eclf3 = eclf3.fit(X, y) >>> print(eclf3.predict(X)) [1 1 1 2 2 2] >>> For more usage examples, please see https://rasbt.github.io/mlxtend/user_guide/classifier/EnsembleVoteClassifier/ ```","title":"EnsembleVoteClassifier"},{"location":"api_modules/mlxtend.classifier/EnsembleVoteClassifier/#methods","text":"fit(X, y, sample_weight=None) Learn weight coefficients from training data for each classifier. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values. sample_weight : array-like, shape = [n_samples], optional Sample weights passed as sample_weights to each regressor in the regressors list as well as the meta_regressor. Raises error if some regressor does not support sample_weight in the fit() method. Returns self : object fit_transform(X, y=None, fit_params) Fit to data, then transform it. Fits transformer to `X` and `y` with optional parameters `fit_params` and returns a transformed version of `X`. Parameters X : array-like of shape (n_samples, n_features) Input samples. y : array-like of shape (n_samples,) or (n_samples, n_outputs), default=None Target values (None for unsupervised transformations). **fit_params : dict Additional fit parameters. Returns X_new : ndarray array of shape (n_samples, n_features_new) Transformed array. get_params(deep=True) Return estimator parameter names for GridSearch support. predict(X) Predict class labels for X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns maj : array-like, shape = [n_samples] Predicted class labels. predict_proba(X) Predict class probabilities for X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns avg : array-like, shape = [n_samples, n_classes] Weighted average probability for each class per sample. score(X, y, sample_weight=None) Return the mean accuracy on the given test data and labels. In multi-label classification, this is the subset accuracy which is a harsh metric since you require for each sample that each label set be correctly predicted. Parameters X : array-like of shape (n_samples, n_features) Test samples. y : array-like of shape (n_samples,) or (n_samples, n_outputs) True labels for X . sample_weight : array-like of shape (n_samples,), default=None Sample weights. Returns score : float Mean accuracy of self.predict(X) w.r.t. y . set_output( , transform=None)* Set output container. See :ref:`sphx_glr_auto_examples_miscellaneous_plot_set_output.py` for an example on how to use the API. Parameters transform : {\"default\", \"pandas\"}, default=None Configure output of transform and fit_transform . \"default\" : Default output format of a transformer \"pandas\" : DataFrame output None : Transform configuration is unchanged Returns self : estimator instance Estimator instance. set_params( params) Set the parameters of this estimator. The method works on simple estimators as well as on nested objects (such as :class:`~sklearn.pipeline.Pipeline`). The latter have parameters of the form ``<component>__<parameter>`` so that it's possible to update each component of a nested object. Parameters **params : dict Estimator parameters. Returns self : estimator instance Estimator instance. transform(X) Return class labels or probabilities for X for each estimator. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns If voting='soft'`` : array-like = [n_classifiers, n_samples, n_classes] Class probabilties calculated by each classifier. If voting='hard'`` : array-like = [n_classifiers, n_samples] Class labels predicted by each classifier.","title":"Methods"},{"location":"api_modules/mlxtend.classifier/LogisticRegression/","text":"LogisticRegression LogisticRegression(eta=0.01, epochs=50, l2_lambda=0.0, minibatches=1, random_seed=None, print_progress=0) Logistic regression classifier. Note that this implementation of Logistic Regression expects binary class labels in {0, 1}. Parameters eta : float (default: 0.01) Learning rate (between 0.0 and 1.0) epochs : int (default: 50) Passes over the training dataset. Prior to each epoch, the dataset is shuffled if minibatches > 1 to prevent cycles in stochastic gradient descent. l2_lambda : float Regularization parameter for L2 regularization. No regularization if l2_lambda=0.0. minibatches : int (default: 1) The number of minibatches for gradient-based optimization. If 1: Gradient Descent learning If len(y): Stochastic Gradient Descent (SGD) online learning If 1 < minibatches < len(y): SGD Minibatch learning random_seed : int (default: None) Set random state for shuffling and initializing the weights. print_progress : int (default: 0) Prints progress in fitting to stderr. 0: No output 1: Epochs elapsed and cost 2: 1 plus time elapsed 3: 2 plus estimated time until completion Attributes w_ : 2d-array, shape={n_features, 1} Model weights after fitting. b_ : 1d-array, shape={1,} Bias unit after fitting. cost_ : list List of floats with cross_entropy cost (sgd or gd) for every epoch. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/classifier/LogisticRegression/ Methods fit(X, y, init_params=True) Learn model from training data. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values. init_params : bool (default: True) Re-initializes model parameters prior to fitting. Set False to continue training with weights from a previous model fitting. Returns self : object get_params(deep=True) Get parameters for this estimator. Parameters deep : boolean, optional If True, will return the parameters for this estimator and contained subobjects that are estimators. Returns params : mapping of string to any Parameter names mapped to their values.' adapted from https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/base.py Author: Gael Varoquaux gael.varoquaux@normalesup.org License: BSD 3 clause predict(X) Predict targets from X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns target_values : array-like, shape = [n_samples] Predicted target values. predict_proba(X) Predict class probabilities of X from the net input. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns Class 1 probability : float score(X, y) Compute the prediction accuracy Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values (true class labels). Returns acc : float The prediction accuracy as a float between 0.0 and 1.0 (perfect score). set_params( params) Set the parameters of this estimator. The method works on simple estimators as well as on nested objects (such as pipelines). The latter have parameters of the form <component>__<parameter> so that it's possible to update each component of a nested object. Returns self adapted from https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/base.py Author: Gael Varoquaux <gael.varoquaux@normalesup.org> License: BSD 3 clause","title":"LogisticRegression"},{"location":"api_modules/mlxtend.classifier/LogisticRegression/#logisticregression","text":"LogisticRegression(eta=0.01, epochs=50, l2_lambda=0.0, minibatches=1, random_seed=None, print_progress=0) Logistic regression classifier. Note that this implementation of Logistic Regression expects binary class labels in {0, 1}. Parameters eta : float (default: 0.01) Learning rate (between 0.0 and 1.0) epochs : int (default: 50) Passes over the training dataset. Prior to each epoch, the dataset is shuffled if minibatches > 1 to prevent cycles in stochastic gradient descent. l2_lambda : float Regularization parameter for L2 regularization. No regularization if l2_lambda=0.0. minibatches : int (default: 1) The number of minibatches for gradient-based optimization. If 1: Gradient Descent learning If len(y): Stochastic Gradient Descent (SGD) online learning If 1 < minibatches < len(y): SGD Minibatch learning random_seed : int (default: None) Set random state for shuffling and initializing the weights. print_progress : int (default: 0) Prints progress in fitting to stderr. 0: No output 1: Epochs elapsed and cost 2: 1 plus time elapsed 3: 2 plus estimated time until completion Attributes w_ : 2d-array, shape={n_features, 1} Model weights after fitting. b_ : 1d-array, shape={1,} Bias unit after fitting. cost_ : list List of floats with cross_entropy cost (sgd or gd) for every epoch. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/classifier/LogisticRegression/","title":"LogisticRegression"},{"location":"api_modules/mlxtend.classifier/LogisticRegression/#methods","text":"fit(X, y, init_params=True) Learn model from training data. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values. init_params : bool (default: True) Re-initializes model parameters prior to fitting. Set False to continue training with weights from a previous model fitting. Returns self : object get_params(deep=True) Get parameters for this estimator. Parameters deep : boolean, optional If True, will return the parameters for this estimator and contained subobjects that are estimators. Returns params : mapping of string to any Parameter names mapped to their values.' adapted from https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/base.py Author: Gael Varoquaux gael.varoquaux@normalesup.org License: BSD 3 clause predict(X) Predict targets from X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns target_values : array-like, shape = [n_samples] Predicted target values. predict_proba(X) Predict class probabilities of X from the net input. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns Class 1 probability : float score(X, y) Compute the prediction accuracy Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values (true class labels). Returns acc : float The prediction accuracy as a float between 0.0 and 1.0 (perfect score). set_params( params) Set the parameters of this estimator. The method works on simple estimators as well as on nested objects (such as pipelines). The latter have parameters of the form <component>__<parameter> so that it's possible to update each component of a nested object. Returns self adapted from https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/base.py Author: Gael Varoquaux <gael.varoquaux@normalesup.org> License: BSD 3 clause","title":"Methods"},{"location":"api_modules/mlxtend.classifier/MultiLayerPerceptron/","text":"MultiLayerPerceptron MultiLayerPerceptron(eta=0.5, epochs=50, hidden_layers=[50], n_classes=None, momentum=0.0, l1=0.0, l2=0.0, dropout=1.0, decrease_const=0.0, minibatches=1, random_seed=None, print_progress=0) Multi-layer perceptron classifier with logistic sigmoid activations Parameters eta : float (default: 0.5) Learning rate (between 0.0 and 1.0) epochs : int (default: 50) Passes over the training dataset. Prior to each epoch, the dataset is shuffled if minibatches > 1 to prevent cycles in stochastic gradient descent. hidden_layers : list (default: [50]) Number of units per hidden layer. By default 50 units in the first hidden layer. At the moment only 1 hidden layer is supported n_classes : int (default: None) A positive integer to declare the number of class labels if not all class labels are present in a partial training set. Gets the number of class labels automatically if None. l1 : float (default: 0.0) L1 regularization strength l2 : float (default: 0.0) L2 regularization strength momentum : float (default: 0.0) Momentum constant. Factor multiplied with the gradient of the previous epoch t-1 to improve learning speed w(t) := w(t) - (grad(t) + momentum * grad(t-1)) decrease_const : float (default: 0.0) Decrease constant. Shrinks the learning rate after each epoch via eta / (1 + epoch*decrease_const) minibatches : int (default: 1) Divide the training data into k minibatches for accelerated stochastic gradient descent learning. Gradient Descent Learning if minibatches = 1 Stochastic Gradient Descent learning if minibatches = len(y) Minibatch learning if minibatches > 1 random_seed : int (default: None) Set random state for shuffling and initializing the weights. print_progress : int (default: 0) Prints progress in fitting to stderr. 0: No output 1: Epochs elapsed and cost 2: 1 plus time elapsed 3: 2 plus estimated time until completion Attributes w_ : 2d-array, shape=[n_features, n_classes] Weights after fitting. b_ : 1D-array, shape=[n_classes] Bias units after fitting. cost_ : list List of floats; the mean categorical cross entropy cost after each epoch. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/classifier/MultiLayerPerceptron/ Methods fit(X, y, init_params=True) Learn model from training data. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values. init_params : bool (default: True) Re-initializes model parameters prior to fitting. Set False to continue training with weights from a previous model fitting. Returns self : object get_params(deep=True) Get parameters for this estimator. Parameters deep : boolean, optional If True, will return the parameters for this estimator and contained subobjects that are estimators. Returns params : mapping of string to any Parameter names mapped to their values.' adapted from https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/base.py Author: Gael Varoquaux gael.varoquaux@normalesup.org License: BSD 3 clause predict(X) Predict targets from X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns target_values : array-like, shape = [n_samples] Predicted target values. predict_proba(X) Predict class probabilities of X from the net input. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns Class probabilties : array-like, shape= [n_samples, n_classes] score(X, y) Compute the prediction accuracy Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values (true class labels). Returns acc : float The prediction accuracy as a float between 0.0 and 1.0 (perfect score). set_params( params) Set the parameters of this estimator. The method works on simple estimators as well as on nested objects (such as pipelines). The latter have parameters of the form <component>__<parameter> so that it's possible to update each component of a nested object. Returns self adapted from https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/base.py Author: Gael Varoquaux <gael.varoquaux@normalesup.org> License: BSD 3 clause","title":"MultiLayerPerceptron"},{"location":"api_modules/mlxtend.classifier/MultiLayerPerceptron/#multilayerperceptron","text":"MultiLayerPerceptron(eta=0.5, epochs=50, hidden_layers=[50], n_classes=None, momentum=0.0, l1=0.0, l2=0.0, dropout=1.0, decrease_const=0.0, minibatches=1, random_seed=None, print_progress=0) Multi-layer perceptron classifier with logistic sigmoid activations Parameters eta : float (default: 0.5) Learning rate (between 0.0 and 1.0) epochs : int (default: 50) Passes over the training dataset. Prior to each epoch, the dataset is shuffled if minibatches > 1 to prevent cycles in stochastic gradient descent. hidden_layers : list (default: [50]) Number of units per hidden layer. By default 50 units in the first hidden layer. At the moment only 1 hidden layer is supported n_classes : int (default: None) A positive integer to declare the number of class labels if not all class labels are present in a partial training set. Gets the number of class labels automatically if None. l1 : float (default: 0.0) L1 regularization strength l2 : float (default: 0.0) L2 regularization strength momentum : float (default: 0.0) Momentum constant. Factor multiplied with the gradient of the previous epoch t-1 to improve learning speed w(t) := w(t) - (grad(t) + momentum * grad(t-1)) decrease_const : float (default: 0.0) Decrease constant. Shrinks the learning rate after each epoch via eta / (1 + epoch*decrease_const) minibatches : int (default: 1) Divide the training data into k minibatches for accelerated stochastic gradient descent learning. Gradient Descent Learning if minibatches = 1 Stochastic Gradient Descent learning if minibatches = len(y) Minibatch learning if minibatches > 1 random_seed : int (default: None) Set random state for shuffling and initializing the weights. print_progress : int (default: 0) Prints progress in fitting to stderr. 0: No output 1: Epochs elapsed and cost 2: 1 plus time elapsed 3: 2 plus estimated time until completion Attributes w_ : 2d-array, shape=[n_features, n_classes] Weights after fitting. b_ : 1D-array, shape=[n_classes] Bias units after fitting. cost_ : list List of floats; the mean categorical cross entropy cost after each epoch. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/classifier/MultiLayerPerceptron/","title":"MultiLayerPerceptron"},{"location":"api_modules/mlxtend.classifier/MultiLayerPerceptron/#methods","text":"fit(X, y, init_params=True) Learn model from training data. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values. init_params : bool (default: True) Re-initializes model parameters prior to fitting. Set False to continue training with weights from a previous model fitting. Returns self : object get_params(deep=True) Get parameters for this estimator. Parameters deep : boolean, optional If True, will return the parameters for this estimator and contained subobjects that are estimators. Returns params : mapping of string to any Parameter names mapped to their values.' adapted from https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/base.py Author: Gael Varoquaux gael.varoquaux@normalesup.org License: BSD 3 clause predict(X) Predict targets from X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns target_values : array-like, shape = [n_samples] Predicted target values. predict_proba(X) Predict class probabilities of X from the net input. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns Class probabilties : array-like, shape= [n_samples, n_classes] score(X, y) Compute the prediction accuracy Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values (true class labels). Returns acc : float The prediction accuracy as a float between 0.0 and 1.0 (perfect score). set_params( params) Set the parameters of this estimator. The method works on simple estimators as well as on nested objects (such as pipelines). The latter have parameters of the form <component>__<parameter> so that it's possible to update each component of a nested object. Returns self adapted from https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/base.py Author: Gael Varoquaux <gael.varoquaux@normalesup.org> License: BSD 3 clause","title":"Methods"},{"location":"api_modules/mlxtend.classifier/OneRClassifier/","text":"OneRClassifier OneRClassifier(resolve_ties='first') OneR (One Rule) Classifier. Parameters resolve_ties : str (default: 'first') Option for how to resolve ties if two or more features have the same error. Options are - 'first' (default): chooses first feature in the list, i.e., feature with the lower column index. - 'chi-squared': performs a chi-squared test for each feature against the target and selects the feature with the lowest p-value. Attributes self.classes_labels_ : array-like, shape = [n_labels] Array containing the unique class labels found in the training set. self.feature_idx_ : int The index of the rules' feature based on the column in the training set. self.p_value_ : float The p value for a given feature. Only available after calling fit when the OneR attribute resolve_ties = 'chi-squared' is set. self.prediction_dict_ : dict Dictionary containing information about the feature's (self.feature_idx_) rules and total error. E.g., {'total error': 37, 'rules (value: class)': {0: 0, 1: 2}} means the total error is 37, and the rules are \"if feature value == 0 classify as 0\" and \"if feature value == 1 classify as 2\". (And classify as class 1 otherwise.) For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/classifier/OneRClassifier/ Methods fit(X, y) Learn rule from training data. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values. Returns self : object get_params(deep=True) Get parameters for this estimator. Parameters deep : bool, default=True If True, will return the parameters for this estimator and contained subobjects that are estimators. Returns params : dict Parameter names mapped to their values. predict(X) Predict class labels for X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns maj : array-like, shape = [n_samples] Predicted class labels. score(X, y, sample_weight=None) Return the mean accuracy on the given test data and labels. In multi-label classification, this is the subset accuracy which is a harsh metric since you require for each sample that each label set be correctly predicted. Parameters X : array-like of shape (n_samples, n_features) Test samples. y : array-like of shape (n_samples,) or (n_samples, n_outputs) True labels for X . sample_weight : array-like of shape (n_samples,), default=None Sample weights. Returns score : float Mean accuracy of self.predict(X) w.r.t. y . set_params( params) Set the parameters of this estimator. The method works on simple estimators as well as on nested objects (such as :class:`~sklearn.pipeline.Pipeline`). The latter have parameters of the form ``<component>__<parameter>`` so that it's possible to update each component of a nested object. Parameters **params : dict Estimator parameters. Returns self : estimator instance Estimator instance.","title":"OneRClassifier"},{"location":"api_modules/mlxtend.classifier/OneRClassifier/#onerclassifier","text":"OneRClassifier(resolve_ties='first') OneR (One Rule) Classifier. Parameters resolve_ties : str (default: 'first') Option for how to resolve ties if two or more features have the same error. Options are - 'first' (default): chooses first feature in the list, i.e., feature with the lower column index. - 'chi-squared': performs a chi-squared test for each feature against the target and selects the feature with the lowest p-value. Attributes self.classes_labels_ : array-like, shape = [n_labels] Array containing the unique class labels found in the training set. self.feature_idx_ : int The index of the rules' feature based on the column in the training set. self.p_value_ : float The p value for a given feature. Only available after calling fit when the OneR attribute resolve_ties = 'chi-squared' is set. self.prediction_dict_ : dict Dictionary containing information about the feature's (self.feature_idx_) rules and total error. E.g., {'total error': 37, 'rules (value: class)': {0: 0, 1: 2}} means the total error is 37, and the rules are \"if feature value == 0 classify as 0\" and \"if feature value == 1 classify as 2\". (And classify as class 1 otherwise.) For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/classifier/OneRClassifier/","title":"OneRClassifier"},{"location":"api_modules/mlxtend.classifier/OneRClassifier/#methods","text":"fit(X, y) Learn rule from training data. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values. Returns self : object get_params(deep=True) Get parameters for this estimator. Parameters deep : bool, default=True If True, will return the parameters for this estimator and contained subobjects that are estimators. Returns params : dict Parameter names mapped to their values. predict(X) Predict class labels for X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns maj : array-like, shape = [n_samples] Predicted class labels. score(X, y, sample_weight=None) Return the mean accuracy on the given test data and labels. In multi-label classification, this is the subset accuracy which is a harsh metric since you require for each sample that each label set be correctly predicted. Parameters X : array-like of shape (n_samples, n_features) Test samples. y : array-like of shape (n_samples,) or (n_samples, n_outputs) True labels for X . sample_weight : array-like of shape (n_samples,), default=None Sample weights. Returns score : float Mean accuracy of self.predict(X) w.r.t. y . set_params( params) Set the parameters of this estimator. The method works on simple estimators as well as on nested objects (such as :class:`~sklearn.pipeline.Pipeline`). The latter have parameters of the form ``<component>__<parameter>`` so that it's possible to update each component of a nested object. Parameters **params : dict Estimator parameters. Returns self : estimator instance Estimator instance.","title":"Methods"},{"location":"api_modules/mlxtend.classifier/Perceptron/","text":"Perceptron Perceptron(eta=0.1, epochs=50, random_seed=None, print_progress=0) Perceptron classifier. Note that this implementation of the Perceptron expects binary class labels in {0, 1}. Parameters eta : float (default: 0.1) Learning rate (between 0.0 and 1.0) epochs : int (default: 50) Number of passes over the training dataset. Prior to each epoch, the dataset is shuffled to prevent cycles. random_seed : int Random state for initializing random weights and shuffling. print_progress : int (default: 0) Prints progress in fitting to stderr. 0: No output 1: Epochs elapsed and cost 2: 1 plus time elapsed 3: 2 plus estimated time until completion Attributes w_ : 2d-array, shape={n_features, 1} Model weights after fitting. b_ : 1d-array, shape={1,} Bias unit after fitting. cost_ : list Number of misclassifications in every epoch. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/classifier/Perceptron/ Methods fit(X, y, init_params=True) Learn model from training data. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values. init_params : bool (default: True) Re-initializes model parameters prior to fitting. Set False to continue training with weights from a previous model fitting. Returns self : object get_params(deep=True) Get parameters for this estimator. Parameters deep : boolean, optional If True, will return the parameters for this estimator and contained subobjects that are estimators. Returns params : mapping of string to any Parameter names mapped to their values.' adapted from https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/base.py Author: Gael Varoquaux gael.varoquaux@normalesup.org License: BSD 3 clause predict(X) Predict targets from X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns target_values : array-like, shape = [n_samples] Predicted target values. score(X, y) Compute the prediction accuracy Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values (true class labels). Returns acc : float The prediction accuracy as a float between 0.0 and 1.0 (perfect score). set_params( params) Set the parameters of this estimator. The method works on simple estimators as well as on nested objects (such as pipelines). The latter have parameters of the form <component>__<parameter> so that it's possible to update each component of a nested object. Returns self adapted from https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/base.py Author: Gael Varoquaux <gael.varoquaux@normalesup.org> License: BSD 3 clause","title":"Perceptron"},{"location":"api_modules/mlxtend.classifier/Perceptron/#perceptron","text":"Perceptron(eta=0.1, epochs=50, random_seed=None, print_progress=0) Perceptron classifier. Note that this implementation of the Perceptron expects binary class labels in {0, 1}. Parameters eta : float (default: 0.1) Learning rate (between 0.0 and 1.0) epochs : int (default: 50) Number of passes over the training dataset. Prior to each epoch, the dataset is shuffled to prevent cycles. random_seed : int Random state for initializing random weights and shuffling. print_progress : int (default: 0) Prints progress in fitting to stderr. 0: No output 1: Epochs elapsed and cost 2: 1 plus time elapsed 3: 2 plus estimated time until completion Attributes w_ : 2d-array, shape={n_features, 1} Model weights after fitting. b_ : 1d-array, shape={1,} Bias unit after fitting. cost_ : list Number of misclassifications in every epoch. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/classifier/Perceptron/","title":"Perceptron"},{"location":"api_modules/mlxtend.classifier/Perceptron/#methods","text":"fit(X, y, init_params=True) Learn model from training data. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values. init_params : bool (default: True) Re-initializes model parameters prior to fitting. Set False to continue training with weights from a previous model fitting. Returns self : object get_params(deep=True) Get parameters for this estimator. Parameters deep : boolean, optional If True, will return the parameters for this estimator and contained subobjects that are estimators. Returns params : mapping of string to any Parameter names mapped to their values.' adapted from https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/base.py Author: Gael Varoquaux gael.varoquaux@normalesup.org License: BSD 3 clause predict(X) Predict targets from X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns target_values : array-like, shape = [n_samples] Predicted target values. score(X, y) Compute the prediction accuracy Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values (true class labels). Returns acc : float The prediction accuracy as a float between 0.0 and 1.0 (perfect score). set_params( params) Set the parameters of this estimator. The method works on simple estimators as well as on nested objects (such as pipelines). The latter have parameters of the form <component>__<parameter> so that it's possible to update each component of a nested object. Returns self adapted from https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/base.py Author: Gael Varoquaux <gael.varoquaux@normalesup.org> License: BSD 3 clause","title":"Methods"},{"location":"api_modules/mlxtend.classifier/SoftmaxRegression/","text":"SoftmaxRegression SoftmaxRegression(eta=0.01, epochs=50, l2=0.0, minibatches=1, n_classes=None, random_seed=None, print_progress=0) Softmax regression classifier. Parameters eta : float (default: 0.01) Learning rate (between 0.0 and 1.0) epochs : int (default: 50) Passes over the training dataset. Prior to each epoch, the dataset is shuffled if minibatches > 1 to prevent cycles in stochastic gradient descent. l2 : float Regularization parameter for L2 regularization. No regularization if l2=0.0. minibatches : int (default: 1) The number of minibatches for gradient-based optimization. If 1: Gradient Descent learning If len(y): Stochastic Gradient Descent (SGD) online learning If 1 < minibatches < len(y): SGD Minibatch learning n_classes : int (default: None) A positive integer to declare the number of class labels if not all class labels are present in a partial training set. Gets the number of class labels automatically if None. random_seed : int (default: None) Set random state for shuffling and initializing the weights. print_progress : int (default: 0) Prints progress in fitting to stderr. 0: No output 1: Epochs elapsed and cost 2: 1 plus time elapsed 3: 2 plus estimated time until completion Attributes w_ : 2d-array, shape={n_features, 1} Model weights after fitting. b_ : 1d-array, shape={1,} Bias unit after fitting. cost_ : list List of floats, the average cross_entropy for each epoch. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/classifier/SoftmaxRegression/ Methods fit(X, y, init_params=True) Learn model from training data. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values. init_params : bool (default: True) Re-initializes model parameters prior to fitting. Set False to continue training with weights from a previous model fitting. Returns self : object get_params(deep=True) Get parameters for this estimator. Parameters deep : boolean, optional If True, will return the parameters for this estimator and contained subobjects that are estimators. Returns params : mapping of string to any Parameter names mapped to their values.' adapted from https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/base.py Author: Gael Varoquaux gael.varoquaux@normalesup.org License: BSD 3 clause predict(X) Predict targets from X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns target_values : array-like, shape = [n_samples] Predicted target values. predict_proba(X) Predict class probabilities of X from the net input. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns Class probabilties : array-like, shape= [n_samples, n_classes] score(X, y) Compute the prediction accuracy Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values (true class labels). Returns acc : float The prediction accuracy as a float between 0.0 and 1.0 (perfect score). set_params( params) Set the parameters of this estimator. The method works on simple estimators as well as on nested objects (such as pipelines). The latter have parameters of the form <component>__<parameter> so that it's possible to update each component of a nested object. Returns self adapted from https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/base.py Author: Gael Varoquaux <gael.varoquaux@normalesup.org> License: BSD 3 clause","title":"SoftmaxRegression"},{"location":"api_modules/mlxtend.classifier/SoftmaxRegression/#softmaxregression","text":"SoftmaxRegression(eta=0.01, epochs=50, l2=0.0, minibatches=1, n_classes=None, random_seed=None, print_progress=0) Softmax regression classifier. Parameters eta : float (default: 0.01) Learning rate (between 0.0 and 1.0) epochs : int (default: 50) Passes over the training dataset. Prior to each epoch, the dataset is shuffled if minibatches > 1 to prevent cycles in stochastic gradient descent. l2 : float Regularization parameter for L2 regularization. No regularization if l2=0.0. minibatches : int (default: 1) The number of minibatches for gradient-based optimization. If 1: Gradient Descent learning If len(y): Stochastic Gradient Descent (SGD) online learning If 1 < minibatches < len(y): SGD Minibatch learning n_classes : int (default: None) A positive integer to declare the number of class labels if not all class labels are present in a partial training set. Gets the number of class labels automatically if None. random_seed : int (default: None) Set random state for shuffling and initializing the weights. print_progress : int (default: 0) Prints progress in fitting to stderr. 0: No output 1: Epochs elapsed and cost 2: 1 plus time elapsed 3: 2 plus estimated time until completion Attributes w_ : 2d-array, shape={n_features, 1} Model weights after fitting. b_ : 1d-array, shape={1,} Bias unit after fitting. cost_ : list List of floats, the average cross_entropy for each epoch. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/classifier/SoftmaxRegression/","title":"SoftmaxRegression"},{"location":"api_modules/mlxtend.classifier/SoftmaxRegression/#methods","text":"fit(X, y, init_params=True) Learn model from training data. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values. init_params : bool (default: True) Re-initializes model parameters prior to fitting. Set False to continue training with weights from a previous model fitting. Returns self : object get_params(deep=True) Get parameters for this estimator. Parameters deep : boolean, optional If True, will return the parameters for this estimator and contained subobjects that are estimators. Returns params : mapping of string to any Parameter names mapped to their values.' adapted from https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/base.py Author: Gael Varoquaux gael.varoquaux@normalesup.org License: BSD 3 clause predict(X) Predict targets from X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns target_values : array-like, shape = [n_samples] Predicted target values. predict_proba(X) Predict class probabilities of X from the net input. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns Class probabilties : array-like, shape= [n_samples, n_classes] score(X, y) Compute the prediction accuracy Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values (true class labels). Returns acc : float The prediction accuracy as a float between 0.0 and 1.0 (perfect score). set_params( params) Set the parameters of this estimator. The method works on simple estimators as well as on nested objects (such as pipelines). The latter have parameters of the form <component>__<parameter> so that it's possible to update each component of a nested object. Returns self adapted from https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/base.py Author: Gael Varoquaux <gael.varoquaux@normalesup.org> License: BSD 3 clause","title":"Methods"},{"location":"api_modules/mlxtend.classifier/StackingCVClassifier/","text":"StackingCVClassifier StackingCVClassifier(classifiers, meta_classifier, use_probas=False, drop_proba_col=None, cv=2, shuffle=True, random_state=None, stratify=True, verbose=0, use_features_in_secondary=False, store_train_meta_features=False, use_clones=True, n_jobs=None, pre_dispatch='2 n_jobs')* A 'Stacking Cross-Validation' classifier for scikit-learn estimators. New in mlxtend v0.4.3 Parameters classifiers : array-like, shape = [n_classifiers] A list of classifiers. Invoking the fit method on the StackingCVClassifer will fit clones of these original classifiers that will be stored in the class attribute self.clfs_ if use_clones=True . meta_classifier : object The meta-classifier to be fitted on the ensemble of classifiers use_probas : bool (default: False) If True, trains meta-classifier based on predicted probabilities instead of class labels. drop_proba_col : string (default: None) Drops extra \"probability\" column in the feature set, because it is redundant: p(y_c) = 1 - p(y_1) + p(y_2) + ... + p(y_{c-1}). This can be useful for meta-classifiers that are sensitive to perfectly collinear features. If 'last', drops last probability column. If 'first', drops first probability column. Only relevant if use_probas=True . cv : int, cross-validation generator or an iterable, optional (default: 2) Determines the cross-validation splitting strategy. Possible inputs for cv are: - None, to use the default 2-fold cross validation, - integer, to specify the number of folds in a (Stratified)KFold , - An object to be used as a cross-validation generator. - An iterable yielding train, test splits. For integer/None inputs, it will use either a KFold or StratifiedKFold cross validation depending the value of stratify argument. shuffle : bool (default: True) If True, and the cv argument is integer, the training data will be shuffled at fitting stage prior to cross-validation. If the cv argument is a specific cross validation technique, this argument is omitted. random_state : int, RandomState instance or None, optional (default: None) Constrols the randomness of the cv splitter. Used when cv is integer and shuffle=True . New in v0.16.0. stratify : bool (default: True) If True, and the cv argument is integer it will follow a stratified K-Fold cross validation technique. If the cv argument is a specific cross validation technique, this argument is omitted. verbose : int, optional (default=0) Controls the verbosity of the building process. - verbose=0 (default): Prints nothing - verbose=1 : Prints the number & name of the regressor being fitted and which fold is currently being used for fitting - verbose=2 : Prints info about the parameters of the regressor being fitted - verbose>2 : Changes verbose param of the underlying regressor to self.verbose - 2 use_features_in_secondary : bool (default: False) If True, the meta-classifier will be trained both on the predictions of the original classifiers and the original dataset. If False, the meta-classifier will be trained only on the predictions of the original classifiers. store_train_meta_features : bool (default: False) If True, the meta-features computed from the training data used for fitting the meta-classifier stored in the self.train_meta_features_ array, which can be accessed after calling fit . use_clones : bool (default: True) Clones the classifiers for stacking classification if True (default) or else uses the original ones, which will be refitted on the dataset upon calling the fit method. Hence, if use_clones=True, the original input classifiers will remain unmodified upon using the StackingCVClassifier's fit method. Setting use_clones=False is recommended if you are working with estimators that are supporting the scikit-learn fit/predict API interface but are not compatible to scikit-learn's clone function. n_jobs : int or None, optional (default=None) The number of CPUs to use to do the computation. None means 1 unless in a joblib.parallel_backend context. -1 means using all processors. for more details. New in v0.16.0. pre_dispatch : int, or string, optional Controls the number of jobs that get dispatched during parallel execution. Reducing this number can be useful to avoid an explosion of memory consumption when more jobs get dispatched than CPUs can process. This parameter can be: - None, in which case all the jobs are immediately created and spawned. Use this for lightweight and fast-running jobs, to avoid delays due to on-demand spawning of the jobs - An int, giving the exact number of total jobs that are spawned - A string, giving an expression as a function of n_jobs, as in '2*n_jobs' New in v0.16.0. Attributes clfs_ : list, shape=[n_classifiers] Fitted classifiers (clones of the original classifiers) meta_clf_ : estimator Fitted meta-classifier (clone of the original meta-estimator) train_meta_features : numpy array, shape = [n_samples, n_classifiers] meta-features for training data, where n_samples is the number of samples in training data and n_classifiers is the number of classfiers. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/classifier/StackingCVClassifier/ Methods decision_function(X) Predict class confidence scores for X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns scores : shape=(n_samples,) if n_classes == 2 else (n_samples, n_classes). Confidence scores per (sample, class) combination. In the binary case, confidence score for self.classes_[1] where >0 means this class would be predicted. fit(X, y, groups=None, sample_weight=None) Fit ensemble classifers and the meta-classifier. Parameters X : numpy array, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : numpy array, shape = [n_samples] Target values. groups : numpy array/None, shape = [n_samples] The group that each sample belongs to. This is used by specific folding strategies such as GroupKFold() sample_weight : array-like, shape = [n_samples], optional Sample weights passed as sample_weights to each regressor in the regressors list as well as the meta_regressor. Raises error if some regressor does not support sample_weight in the fit() method. Returns self : object fit_transform(X, y=None, fit_params) Fit to data, then transform it. Fits transformer to `X` and `y` with optional parameters `fit_params` and returns a transformed version of `X`. Parameters X : array-like of shape (n_samples, n_features) Input samples. y : array-like of shape (n_samples,) or (n_samples, n_outputs), default=None Target values (None for unsupervised transformations). **fit_params : dict Additional fit parameters. Returns X_new : ndarray array of shape (n_samples, n_features_new) Transformed array. get_params(deep=True) Return estimator parameter names for GridSearch support. predict(X) Predict target values for X. Parameters X : numpy array, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns labels : array-like, shape = [n_samples] Predicted class labels. predict_meta_features(X) Get meta-features of test-data. Parameters X : numpy array, shape = [n_samples, n_features] Test vectors, where n_samples is the number of samples and n_features is the number of features. Returns meta-features : numpy array, shape = [n_samples, n_classifiers] Returns the meta-features for test data. predict_proba(X) Predict class probabilities for X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns proba : array-like, shape = [n_samples, n_classes] or a list of n_outputs of such arrays if n_outputs > 1. Probability for each class per sample. score(X, y, sample_weight=None) Return the mean accuracy on the given test data and labels. In multi-label classification, this is the subset accuracy which is a harsh metric since you require for each sample that each label set be correctly predicted. Parameters X : array-like of shape (n_samples, n_features) Test samples. y : array-like of shape (n_samples,) or (n_samples, n_outputs) True labels for X . sample_weight : array-like of shape (n_samples,), default=None Sample weights. Returns score : float Mean accuracy of self.predict(X) w.r.t. y . set_output( , transform=None)* Set output container. See :ref:`sphx_glr_auto_examples_miscellaneous_plot_set_output.py` for an example on how to use the API. Parameters transform : {\"default\", \"pandas\"}, default=None Configure output of transform and fit_transform . \"default\" : Default output format of a transformer \"pandas\" : DataFrame output None : Transform configuration is unchanged Returns self : estimator instance Estimator instance. set_params( params) Set the parameters of this estimator. Valid parameter keys can be listed with ``get_params()``. Returns self Properties named_classifiers None","title":"StackingCVClassifier"},{"location":"api_modules/mlxtend.classifier/StackingCVClassifier/#stackingcvclassifier","text":"StackingCVClassifier(classifiers, meta_classifier, use_probas=False, drop_proba_col=None, cv=2, shuffle=True, random_state=None, stratify=True, verbose=0, use_features_in_secondary=False, store_train_meta_features=False, use_clones=True, n_jobs=None, pre_dispatch='2 n_jobs')* A 'Stacking Cross-Validation' classifier for scikit-learn estimators. New in mlxtend v0.4.3 Parameters classifiers : array-like, shape = [n_classifiers] A list of classifiers. Invoking the fit method on the StackingCVClassifer will fit clones of these original classifiers that will be stored in the class attribute self.clfs_ if use_clones=True . meta_classifier : object The meta-classifier to be fitted on the ensemble of classifiers use_probas : bool (default: False) If True, trains meta-classifier based on predicted probabilities instead of class labels. drop_proba_col : string (default: None) Drops extra \"probability\" column in the feature set, because it is redundant: p(y_c) = 1 - p(y_1) + p(y_2) + ... + p(y_{c-1}). This can be useful for meta-classifiers that are sensitive to perfectly collinear features. If 'last', drops last probability column. If 'first', drops first probability column. Only relevant if use_probas=True . cv : int, cross-validation generator or an iterable, optional (default: 2) Determines the cross-validation splitting strategy. Possible inputs for cv are: - None, to use the default 2-fold cross validation, - integer, to specify the number of folds in a (Stratified)KFold , - An object to be used as a cross-validation generator. - An iterable yielding train, test splits. For integer/None inputs, it will use either a KFold or StratifiedKFold cross validation depending the value of stratify argument. shuffle : bool (default: True) If True, and the cv argument is integer, the training data will be shuffled at fitting stage prior to cross-validation. If the cv argument is a specific cross validation technique, this argument is omitted. random_state : int, RandomState instance or None, optional (default: None) Constrols the randomness of the cv splitter. Used when cv is integer and shuffle=True . New in v0.16.0. stratify : bool (default: True) If True, and the cv argument is integer it will follow a stratified K-Fold cross validation technique. If the cv argument is a specific cross validation technique, this argument is omitted. verbose : int, optional (default=0) Controls the verbosity of the building process. - verbose=0 (default): Prints nothing - verbose=1 : Prints the number & name of the regressor being fitted and which fold is currently being used for fitting - verbose=2 : Prints info about the parameters of the regressor being fitted - verbose>2 : Changes verbose param of the underlying regressor to self.verbose - 2 use_features_in_secondary : bool (default: False) If True, the meta-classifier will be trained both on the predictions of the original classifiers and the original dataset. If False, the meta-classifier will be trained only on the predictions of the original classifiers. store_train_meta_features : bool (default: False) If True, the meta-features computed from the training data used for fitting the meta-classifier stored in the self.train_meta_features_ array, which can be accessed after calling fit . use_clones : bool (default: True) Clones the classifiers for stacking classification if True (default) or else uses the original ones, which will be refitted on the dataset upon calling the fit method. Hence, if use_clones=True, the original input classifiers will remain unmodified upon using the StackingCVClassifier's fit method. Setting use_clones=False is recommended if you are working with estimators that are supporting the scikit-learn fit/predict API interface but are not compatible to scikit-learn's clone function. n_jobs : int or None, optional (default=None) The number of CPUs to use to do the computation. None means 1 unless in a joblib.parallel_backend context. -1 means using all processors. for more details. New in v0.16.0. pre_dispatch : int, or string, optional Controls the number of jobs that get dispatched during parallel execution. Reducing this number can be useful to avoid an explosion of memory consumption when more jobs get dispatched than CPUs can process. This parameter can be: - None, in which case all the jobs are immediately created and spawned. Use this for lightweight and fast-running jobs, to avoid delays due to on-demand spawning of the jobs - An int, giving the exact number of total jobs that are spawned - A string, giving an expression as a function of n_jobs, as in '2*n_jobs' New in v0.16.0. Attributes clfs_ : list, shape=[n_classifiers] Fitted classifiers (clones of the original classifiers) meta_clf_ : estimator Fitted meta-classifier (clone of the original meta-estimator) train_meta_features : numpy array, shape = [n_samples, n_classifiers] meta-features for training data, where n_samples is the number of samples in training data and n_classifiers is the number of classfiers. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/classifier/StackingCVClassifier/","title":"StackingCVClassifier"},{"location":"api_modules/mlxtend.classifier/StackingCVClassifier/#methods","text":"decision_function(X) Predict class confidence scores for X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns scores : shape=(n_samples,) if n_classes == 2 else (n_samples, n_classes). Confidence scores per (sample, class) combination. In the binary case, confidence score for self.classes_[1] where >0 means this class would be predicted. fit(X, y, groups=None, sample_weight=None) Fit ensemble classifers and the meta-classifier. Parameters X : numpy array, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : numpy array, shape = [n_samples] Target values. groups : numpy array/None, shape = [n_samples] The group that each sample belongs to. This is used by specific folding strategies such as GroupKFold() sample_weight : array-like, shape = [n_samples], optional Sample weights passed as sample_weights to each regressor in the regressors list as well as the meta_regressor. Raises error if some regressor does not support sample_weight in the fit() method. Returns self : object fit_transform(X, y=None, fit_params) Fit to data, then transform it. Fits transformer to `X` and `y` with optional parameters `fit_params` and returns a transformed version of `X`. Parameters X : array-like of shape (n_samples, n_features) Input samples. y : array-like of shape (n_samples,) or (n_samples, n_outputs), default=None Target values (None for unsupervised transformations). **fit_params : dict Additional fit parameters. Returns X_new : ndarray array of shape (n_samples, n_features_new) Transformed array. get_params(deep=True) Return estimator parameter names for GridSearch support. predict(X) Predict target values for X. Parameters X : numpy array, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns labels : array-like, shape = [n_samples] Predicted class labels. predict_meta_features(X) Get meta-features of test-data. Parameters X : numpy array, shape = [n_samples, n_features] Test vectors, where n_samples is the number of samples and n_features is the number of features. Returns meta-features : numpy array, shape = [n_samples, n_classifiers] Returns the meta-features for test data. predict_proba(X) Predict class probabilities for X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns proba : array-like, shape = [n_samples, n_classes] or a list of n_outputs of such arrays if n_outputs > 1. Probability for each class per sample. score(X, y, sample_weight=None) Return the mean accuracy on the given test data and labels. In multi-label classification, this is the subset accuracy which is a harsh metric since you require for each sample that each label set be correctly predicted. Parameters X : array-like of shape (n_samples, n_features) Test samples. y : array-like of shape (n_samples,) or (n_samples, n_outputs) True labels for X . sample_weight : array-like of shape (n_samples,), default=None Sample weights. Returns score : float Mean accuracy of self.predict(X) w.r.t. y . set_output( , transform=None)* Set output container. See :ref:`sphx_glr_auto_examples_miscellaneous_plot_set_output.py` for an example on how to use the API. Parameters transform : {\"default\", \"pandas\"}, default=None Configure output of transform and fit_transform . \"default\" : Default output format of a transformer \"pandas\" : DataFrame output None : Transform configuration is unchanged Returns self : estimator instance Estimator instance. set_params( params) Set the parameters of this estimator. Valid parameter keys can be listed with ``get_params()``. Returns self","title":"Methods"},{"location":"api_modules/mlxtend.classifier/StackingCVClassifier/#properties","text":"named_classifiers None","title":"Properties"},{"location":"api_modules/mlxtend.classifier/StackingClassifier/","text":"StackingClassifier StackingClassifier(classifiers, meta_classifier, use_probas=False, drop_proba_col=None, average_probas=False, verbose=0, use_features_in_secondary=False, store_train_meta_features=False, use_clones=True, fit_base_estimators=True) A Stacking classifier for scikit-learn estimators for classification. Parameters classifiers : array-like, shape = [n_classifiers] A list of classifiers. Invoking the fit method on the StackingClassifer will fit clones of these original classifiers that will be stored in the class attribute self.clfs_ if use_clones=True (default) and fit_base_estimators=True (default). meta_classifier : object The meta-classifier to be fitted on the ensemble of classifiers use_probas : bool (default: False) If True, trains meta-classifier based on predicted probabilities instead of class labels. drop_proba_col : string (default: None) Drops extra \"probability\" column in the feature set, because it is redundant: p(y_c) = 1 - p(y_1) + p(y_2) + ... + p(y_{c-1}). This can be useful for meta-classifiers that are sensitive to perfectly collinear features. If 'last', drops last probability column. If 'first', drops first probability column. Only relevant if use_probas=True . average_probas : bool (default: False) Averages the probabilities as meta features if True . Only relevant if use_probas=True . verbose : int, optional (default=0) Controls the verbosity of the building process. - verbose=0 (default): Prints nothing - verbose=1 : Prints the number & name of the regressor being fitted - verbose=2 : Prints info about the parameters of the regressor being fitted - verbose>2 : Changes verbose param of the underlying regressor to self.verbose - 2 use_features_in_secondary : bool (default: False) If True, the meta-classifier will be trained both on the predictions of the original classifiers and the original dataset. If False, the meta-classifier will be trained only on the predictions of the original classifiers. store_train_meta_features : bool (default: False) If True, the meta-features computed from the training data used for fitting the meta-classifier stored in the self.train_meta_features_ array, which can be accessed after calling fit . use_clones : bool (default: True) Clones the classifiers for stacking classification if True (default) or else uses the original ones, which will be refitted on the dataset upon calling the fit method. Hence, if use_clones=True, the original input classifiers will remain unmodified upon using the StackingClassifier's fit method. Setting use_clones=False is recommended if you are working with estimators that are supporting the scikit-learn fit/predict API interface but are not compatible to scikit-learn's clone function. fit_base_estimators: bool (default: True) Refits classifiers in classifiers if True; uses references to the classifiers , otherwise (assumes that the classifiers were already fit). Note: fit_base_estimators=False will enforce use_clones to be False, and is incompatible to most scikit-learn wrappers! For instance, if any form of cross-validation is performed this would require the re-fitting classifiers to training folds, which would raise a NotFitterError if fit_base_estimators=False. (New in mlxtend v0.6.) Attributes clfs_ : list, shape=[n_classifiers] Fitted classifiers (clones of the original classifiers) meta_clf_ : estimator Fitted meta-classifier (clone of the original meta-estimator) train_meta_features : numpy array, shape = [n_samples, n_classifiers] meta-features for training data, where n_samples is the number of samples in training data and n_classifiers is the number of classfiers. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/classifier/StackingClassifier/ Methods decision_function(X) Predict class confidence scores for X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns scores : shape=(n_samples,) if n_classes == 2 else (n_samples, n_classes). Confidence scores per (sample, class) combination. In the binary case, confidence score for self.classes_[1] where >0 means this class would be predicted. fit(X, y, sample_weight=None) Fit ensemble classifers and the meta-classifier. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] or [n_samples, n_outputs] Target values. sample_weight : array-like, shape = [n_samples], optional Sample weights passed as sample_weights to each regressor in the regressors list as well as the meta_regressor. Raises error if some regressor does not support sample_weight in the fit() method. Returns self : object fit_transform(X, y=None, fit_params) Fit to data, then transform it. Fits transformer to `X` and `y` with optional parameters `fit_params` and returns a transformed version of `X`. Parameters X : array-like of shape (n_samples, n_features) Input samples. y : array-like of shape (n_samples,) or (n_samples, n_outputs), default=None Target values (None for unsupervised transformations). **fit_params : dict Additional fit parameters. Returns X_new : ndarray array of shape (n_samples, n_features_new) Transformed array. get_params(deep=True) Return estimator parameter names for GridSearch support. predict(X) Predict target values for X. Parameters X : numpy array, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns labels : array-like, shape = [n_samples] Predicted class labels. predict_meta_features(X) Get meta-features of test-data. Parameters X : numpy array, shape = [n_samples, n_features] Test vectors, where n_samples is the number of samples and n_features is the number of features. Returns meta-features : numpy array, shape = [n_samples, n_classifiers] Returns the meta-features for test data. predict_proba(X) Predict class probabilities for X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns proba : array-like, shape = [n_samples, n_classes] or a list of n_outputs of such arrays if n_outputs > 1. Probability for each class per sample. score(X, y, sample_weight=None) Return the mean accuracy on the given test data and labels. In multi-label classification, this is the subset accuracy which is a harsh metric since you require for each sample that each label set be correctly predicted. Parameters X : array-like of shape (n_samples, n_features) Test samples. y : array-like of shape (n_samples,) or (n_samples, n_outputs) True labels for X . sample_weight : array-like of shape (n_samples,), default=None Sample weights. Returns score : float Mean accuracy of self.predict(X) w.r.t. y . set_output( , transform=None)* Set output container. See :ref:`sphx_glr_auto_examples_miscellaneous_plot_set_output.py` for an example on how to use the API. Parameters transform : {\"default\", \"pandas\"}, default=None Configure output of transform and fit_transform . \"default\" : Default output format of a transformer \"pandas\" : DataFrame output None : Transform configuration is unchanged Returns self : estimator instance Estimator instance. set_params( params) Set the parameters of this estimator. Valid parameter keys can be listed with ``get_params()``. Returns self Properties named_classifiers None","title":"StackingClassifier"},{"location":"api_modules/mlxtend.classifier/StackingClassifier/#stackingclassifier","text":"StackingClassifier(classifiers, meta_classifier, use_probas=False, drop_proba_col=None, average_probas=False, verbose=0, use_features_in_secondary=False, store_train_meta_features=False, use_clones=True, fit_base_estimators=True) A Stacking classifier for scikit-learn estimators for classification. Parameters classifiers : array-like, shape = [n_classifiers] A list of classifiers. Invoking the fit method on the StackingClassifer will fit clones of these original classifiers that will be stored in the class attribute self.clfs_ if use_clones=True (default) and fit_base_estimators=True (default). meta_classifier : object The meta-classifier to be fitted on the ensemble of classifiers use_probas : bool (default: False) If True, trains meta-classifier based on predicted probabilities instead of class labels. drop_proba_col : string (default: None) Drops extra \"probability\" column in the feature set, because it is redundant: p(y_c) = 1 - p(y_1) + p(y_2) + ... + p(y_{c-1}). This can be useful for meta-classifiers that are sensitive to perfectly collinear features. If 'last', drops last probability column. If 'first', drops first probability column. Only relevant if use_probas=True . average_probas : bool (default: False) Averages the probabilities as meta features if True . Only relevant if use_probas=True . verbose : int, optional (default=0) Controls the verbosity of the building process. - verbose=0 (default): Prints nothing - verbose=1 : Prints the number & name of the regressor being fitted - verbose=2 : Prints info about the parameters of the regressor being fitted - verbose>2 : Changes verbose param of the underlying regressor to self.verbose - 2 use_features_in_secondary : bool (default: False) If True, the meta-classifier will be trained both on the predictions of the original classifiers and the original dataset. If False, the meta-classifier will be trained only on the predictions of the original classifiers. store_train_meta_features : bool (default: False) If True, the meta-features computed from the training data used for fitting the meta-classifier stored in the self.train_meta_features_ array, which can be accessed after calling fit . use_clones : bool (default: True) Clones the classifiers for stacking classification if True (default) or else uses the original ones, which will be refitted on the dataset upon calling the fit method. Hence, if use_clones=True, the original input classifiers will remain unmodified upon using the StackingClassifier's fit method. Setting use_clones=False is recommended if you are working with estimators that are supporting the scikit-learn fit/predict API interface but are not compatible to scikit-learn's clone function. fit_base_estimators: bool (default: True) Refits classifiers in classifiers if True; uses references to the classifiers , otherwise (assumes that the classifiers were already fit). Note: fit_base_estimators=False will enforce use_clones to be False, and is incompatible to most scikit-learn wrappers! For instance, if any form of cross-validation is performed this would require the re-fitting classifiers to training folds, which would raise a NotFitterError if fit_base_estimators=False. (New in mlxtend v0.6.) Attributes clfs_ : list, shape=[n_classifiers] Fitted classifiers (clones of the original classifiers) meta_clf_ : estimator Fitted meta-classifier (clone of the original meta-estimator) train_meta_features : numpy array, shape = [n_samples, n_classifiers] meta-features for training data, where n_samples is the number of samples in training data and n_classifiers is the number of classfiers. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/classifier/StackingClassifier/","title":"StackingClassifier"},{"location":"api_modules/mlxtend.classifier/StackingClassifier/#methods","text":"decision_function(X) Predict class confidence scores for X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns scores : shape=(n_samples,) if n_classes == 2 else (n_samples, n_classes). Confidence scores per (sample, class) combination. In the binary case, confidence score for self.classes_[1] where >0 means this class would be predicted. fit(X, y, sample_weight=None) Fit ensemble classifers and the meta-classifier. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] or [n_samples, n_outputs] Target values. sample_weight : array-like, shape = [n_samples], optional Sample weights passed as sample_weights to each regressor in the regressors list as well as the meta_regressor. Raises error if some regressor does not support sample_weight in the fit() method. Returns self : object fit_transform(X, y=None, fit_params) Fit to data, then transform it. Fits transformer to `X` and `y` with optional parameters `fit_params` and returns a transformed version of `X`. Parameters X : array-like of shape (n_samples, n_features) Input samples. y : array-like of shape (n_samples,) or (n_samples, n_outputs), default=None Target values (None for unsupervised transformations). **fit_params : dict Additional fit parameters. Returns X_new : ndarray array of shape (n_samples, n_features_new) Transformed array. get_params(deep=True) Return estimator parameter names for GridSearch support. predict(X) Predict target values for X. Parameters X : numpy array, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns labels : array-like, shape = [n_samples] Predicted class labels. predict_meta_features(X) Get meta-features of test-data. Parameters X : numpy array, shape = [n_samples, n_features] Test vectors, where n_samples is the number of samples and n_features is the number of features. Returns meta-features : numpy array, shape = [n_samples, n_classifiers] Returns the meta-features for test data. predict_proba(X) Predict class probabilities for X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns proba : array-like, shape = [n_samples, n_classes] or a list of n_outputs of such arrays if n_outputs > 1. Probability for each class per sample. score(X, y, sample_weight=None) Return the mean accuracy on the given test data and labels. In multi-label classification, this is the subset accuracy which is a harsh metric since you require for each sample that each label set be correctly predicted. Parameters X : array-like of shape (n_samples, n_features) Test samples. y : array-like of shape (n_samples,) or (n_samples, n_outputs) True labels for X . sample_weight : array-like of shape (n_samples,), default=None Sample weights. Returns score : float Mean accuracy of self.predict(X) w.r.t. y . set_output( , transform=None)* Set output container. See :ref:`sphx_glr_auto_examples_miscellaneous_plot_set_output.py` for an example on how to use the API. Parameters transform : {\"default\", \"pandas\"}, default=None Configure output of transform and fit_transform . \"default\" : Default output format of a transformer \"pandas\" : DataFrame output None : Transform configuration is unchanged Returns self : estimator instance Estimator instance. set_params( params) Set the parameters of this estimator. Valid parameter keys can be listed with ``get_params()``. Returns self","title":"Methods"},{"location":"api_modules/mlxtend.classifier/StackingClassifier/#properties","text":"named_classifiers None","title":"Properties"},{"location":"api_modules/mlxtend.cluster/Kmeans/","text":"Kmeans Kmeans(k, max_iter=10, convergence_tolerance=1e-05, random_seed=None, print_progress=0) K-means clustering class. Added in 0.4.1dev Parameters k : int Number of clusters max_iter : int (default: 10) Number of iterations during cluster assignment. Cluster re-assignment stops automatically when the algorithm converged. convergence_tolerance : float (default: 1e-05) Compares current centroids with centroids of the previous iteration using the given tolerance (a small positive float)to determine if the algorithm converged early. random_seed : int (default: None) Set random state for the initial centroid assignment. print_progress : int (default: 0) Prints progress in fitting to stderr. 0: No output 1: Iterations elapsed 2: 1 plus time elapsed 3: 2 plus estimated time until completion Attributes centroids_ : 2d-array, shape={k, n_features} Feature values of the k cluster centroids. custers_ : dictionary The cluster assignments stored as a Python dictionary; the dictionary keys denote the cluster indeces and the items are Python lists of the sample indices that were assigned to each cluster. iterations_ : int Number of iterations until convergence. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/classifier/Kmeans/ Methods fit(X, init_params=True) Learn model from training data. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. init_params : bool (default: True) Re-initializes model parameters prior to fitting. Set False to continue training with weights from a previous model fitting. Returns self : object get_params(deep=True) Get parameters for this estimator. Parameters deep : boolean, optional If True, will return the parameters for this estimator and contained subobjects that are estimators. Returns params : mapping of string to any Parameter names mapped to their values.' adapted from https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/base.py Author: Gael Varoquaux gael.varoquaux@normalesup.org License: BSD 3 clause predict(X) Predict targets from X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns target_values : array-like, shape = [n_samples] Predicted target values. set_params( params) Set the parameters of this estimator. The method works on simple estimators as well as on nested objects (such as pipelines). The latter have parameters of the form <component>__<parameter> so that it's possible to update each component of a nested object. Returns self adapted from https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/base.py Author: Gael Varoquaux <gael.varoquaux@normalesup.org> License: BSD 3 clause","title":"Kmeans"},{"location":"api_modules/mlxtend.cluster/Kmeans/#kmeans","text":"Kmeans(k, max_iter=10, convergence_tolerance=1e-05, random_seed=None, print_progress=0) K-means clustering class. Added in 0.4.1dev Parameters k : int Number of clusters max_iter : int (default: 10) Number of iterations during cluster assignment. Cluster re-assignment stops automatically when the algorithm converged. convergence_tolerance : float (default: 1e-05) Compares current centroids with centroids of the previous iteration using the given tolerance (a small positive float)to determine if the algorithm converged early. random_seed : int (default: None) Set random state for the initial centroid assignment. print_progress : int (default: 0) Prints progress in fitting to stderr. 0: No output 1: Iterations elapsed 2: 1 plus time elapsed 3: 2 plus estimated time until completion Attributes centroids_ : 2d-array, shape={k, n_features} Feature values of the k cluster centroids. custers_ : dictionary The cluster assignments stored as a Python dictionary; the dictionary keys denote the cluster indeces and the items are Python lists of the sample indices that were assigned to each cluster. iterations_ : int Number of iterations until convergence. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/classifier/Kmeans/","title":"Kmeans"},{"location":"api_modules/mlxtend.cluster/Kmeans/#methods","text":"fit(X, init_params=True) Learn model from training data. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. init_params : bool (default: True) Re-initializes model parameters prior to fitting. Set False to continue training with weights from a previous model fitting. Returns self : object get_params(deep=True) Get parameters for this estimator. Parameters deep : boolean, optional If True, will return the parameters for this estimator and contained subobjects that are estimators. Returns params : mapping of string to any Parameter names mapped to their values.' adapted from https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/base.py Author: Gael Varoquaux gael.varoquaux@normalesup.org License: BSD 3 clause predict(X) Predict targets from X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns target_values : array-like, shape = [n_samples] Predicted target values. set_params( params) Set the parameters of this estimator. The method works on simple estimators as well as on nested objects (such as pipelines). The latter have parameters of the form <component>__<parameter> so that it's possible to update each component of a nested object. Returns self adapted from https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/base.py Author: Gael Varoquaux <gael.varoquaux@normalesup.org> License: BSD 3 clause","title":"Methods"},{"location":"api_modules/mlxtend.data/autompg_data/","text":"autompg_data autompg_data() Auto MPG dataset. Source : https://archive.ics.uci.edu/ml/datasets/Auto+MPG Number of samples : 392 Continuous target variable : mpg Dataset Attributes: 1) cylinders: multi-valued discrete 2) displacement: continuous 3) horsepower: continuous 4) weight: continuous 5) acceleration: continuous 6) model year: multi-valued discrete 7) origin: multi-valued discrete 8) car name: string (unique for each instance) Returns X, y : [n_samples, n_features], [n_targets] X is the feature matrix with 392 auto samples as rows and 8 feature columns (6 rows with NaNs removed). y is a 1-dimensional array of the target MPG values. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/data/autompg_data/","title":"Autompg data"},{"location":"api_modules/mlxtend.data/autompg_data/#autompg_data","text":"autompg_data() Auto MPG dataset. Source : https://archive.ics.uci.edu/ml/datasets/Auto+MPG Number of samples : 392 Continuous target variable : mpg Dataset Attributes: 1) cylinders: multi-valued discrete 2) displacement: continuous 3) horsepower: continuous 4) weight: continuous 5) acceleration: continuous 6) model year: multi-valued discrete 7) origin: multi-valued discrete 8) car name: string (unique for each instance) Returns X, y : [n_samples, n_features], [n_targets] X is the feature matrix with 392 auto samples as rows and 8 feature columns (6 rows with NaNs removed). y is a 1-dimensional array of the target MPG values. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/data/autompg_data/","title":"autompg_data"},{"location":"api_modules/mlxtend.data/boston_housing_data/","text":"boston_housing_data boston_housing_data() Boston Housing dataset. Source : https://archive.ics.uci.edu/ml/datasets/Housing Number of samples : 506 Continuous target variable : MEDV MEDV = Median value of owner-occupied homes in $1000's Dataset Attributes: 1) CRIM per capita crime rate by town 2) ZN proportion of residential land zoned for lots over 25,000 sq.ft. 3) INDUS proportion of non-retail business acres per town 4) CHAS Charles River dummy variable (= 1 if tract bounds river; 0 otherwise) 5) NOX nitric oxides concentration (parts per 10 million) 6) RM average number of rooms per dwelling 7) AGE proportion of owner-occupied units built prior to 1940 8) DIS weighted distances to five Boston employment centres 9) RAD index of accessibility to radial highways 10) TAX full-value property-tax rate per $10,000 11) PTRATIO pupil-teacher ratio by town 12) B 1000(Bk - 0.63)^2 where Bk is the prop. of b. by town 13) LSTAT % lower status of the population Returns X, y : [n_samples, n_features], [n_class_labels] X is the feature matrix with 506 housing samples as rows and 13 feature columns. y is a 1-dimensional array of the continuous target variable MEDV Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/data/boston_housing_data/","title":"Boston housing data"},{"location":"api_modules/mlxtend.data/boston_housing_data/#boston_housing_data","text":"boston_housing_data() Boston Housing dataset. Source : https://archive.ics.uci.edu/ml/datasets/Housing Number of samples : 506 Continuous target variable : MEDV MEDV = Median value of owner-occupied homes in $1000's Dataset Attributes: 1) CRIM per capita crime rate by town 2) ZN proportion of residential land zoned for lots over 25,000 sq.ft. 3) INDUS proportion of non-retail business acres per town 4) CHAS Charles River dummy variable (= 1 if tract bounds river; 0 otherwise) 5) NOX nitric oxides concentration (parts per 10 million) 6) RM average number of rooms per dwelling 7) AGE proportion of owner-occupied units built prior to 1940 8) DIS weighted distances to five Boston employment centres 9) RAD index of accessibility to radial highways 10) TAX full-value property-tax rate per $10,000 11) PTRATIO pupil-teacher ratio by town 12) B 1000(Bk - 0.63)^2 where Bk is the prop. of b. by town 13) LSTAT % lower status of the population Returns X, y : [n_samples, n_features], [n_class_labels] X is the feature matrix with 506 housing samples as rows and 13 feature columns. y is a 1-dimensional array of the continuous target variable MEDV Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/data/boston_housing_data/","title":"boston_housing_data"},{"location":"api_modules/mlxtend.data/iris_data/","text":"iris_data iris_data(version='uci') Iris flower dataset. Source : https://archive.ics.uci.edu/ml/datasets/Iris Number of samples : 150 Class labels : {0, 1, 2}, distribution: [50, 50, 50] 0 = setosa, 1 = versicolor, 2 = virginica. Dataset Attributes: 1) sepal length [cm] 2) sepal width [cm] 3) petal length [cm] 4) petal width [cm] Parameters version : string, optional (default: 'uci'). Version to use {'uci', 'corrected'}. 'uci' loads the dataset as deposited on the UCI machine learning repository, and 'corrected' provides the version that is consistent with Fisher's original paper. See Note for details. Returns X, y : [n_samples, n_features], [n_class_labels] X is the feature matrix with 150 flower samples as rows, and 4 feature columns sepal length, sepal width, petal length, and petal width. y is a 1-dimensional array of the class labels {0, 1, 2} Note The Iris dataset (originally collected by Edgar Anderson) and available in UCI's machine learning repository is different from the Iris dataset described in the original paper by R.A. Fisher [1]). Precisely, there are two data points (row number 34 and 37) in UCI's Machine Learning repository are different from the origianlly published Iris dataset. Also, the original version of the Iris Dataset, which can be loaded via version='corrected' is the same as the one in R. [1] . A. Fisher (1936). \"The use of multiple measurements in taxonomic problems\". Annals of Eugenics. 7 (2): 179\u2013188 Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/data/iris_data/","title":"Iris data"},{"location":"api_modules/mlxtend.data/iris_data/#iris_data","text":"iris_data(version='uci') Iris flower dataset. Source : https://archive.ics.uci.edu/ml/datasets/Iris Number of samples : 150 Class labels : {0, 1, 2}, distribution: [50, 50, 50] 0 = setosa, 1 = versicolor, 2 = virginica. Dataset Attributes: 1) sepal length [cm] 2) sepal width [cm] 3) petal length [cm] 4) petal width [cm] Parameters version : string, optional (default: 'uci'). Version to use {'uci', 'corrected'}. 'uci' loads the dataset as deposited on the UCI machine learning repository, and 'corrected' provides the version that is consistent with Fisher's original paper. See Note for details. Returns X, y : [n_samples, n_features], [n_class_labels] X is the feature matrix with 150 flower samples as rows, and 4 feature columns sepal length, sepal width, petal length, and petal width. y is a 1-dimensional array of the class labels {0, 1, 2} Note The Iris dataset (originally collected by Edgar Anderson) and available in UCI's machine learning repository is different from the Iris dataset described in the original paper by R.A. Fisher [1]). Precisely, there are two data points (row number 34 and 37) in UCI's Machine Learning repository are different from the origianlly published Iris dataset. Also, the original version of the Iris Dataset, which can be loaded via version='corrected' is the same as the one in R. [1] . A. Fisher (1936). \"The use of multiple measurements in taxonomic problems\". Annals of Eugenics. 7 (2): 179\u2013188 Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/data/iris_data/","title":"iris_data"},{"location":"api_modules/mlxtend.data/loadlocal_mnist/","text":"loadlocal_mnist loadlocal_mnist(images_path, labels_path) Read MNIST from ubyte files. Parameters images_path : str path to the test or train MNIST ubyte file labels_path : str path to the test or train MNIST class labels file Returns images : [n_samples, n_pixels] numpy.array Pixel values of the images. labels : [n_samples] numpy array Target class labels Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/data/loadlocal_mnist/","title":"Loadlocal mnist"},{"location":"api_modules/mlxtend.data/loadlocal_mnist/#loadlocal_mnist","text":"loadlocal_mnist(images_path, labels_path) Read MNIST from ubyte files. Parameters images_path : str path to the test or train MNIST ubyte file labels_path : str path to the test or train MNIST class labels file Returns images : [n_samples, n_pixels] numpy.array Pixel values of the images. labels : [n_samples] numpy array Target class labels Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/data/loadlocal_mnist/","title":"loadlocal_mnist"},{"location":"api_modules/mlxtend.data/make_multiplexer_dataset/","text":"make_multiplexer_dataset make_multiplexer_dataset(address_bits=2, sample_size=100, positive_class_ratio=0.5, shuffle=False, random_seed=None) Function to create a binary n-bit multiplexer dataset. New in mlxtend v0.9 Parameters address_bits : int (default: 2) A positive integer that determines the number of address bits in the multiplexer, which in turn determine the n-bit capacity of the multiplexer and therefore the number of features. The number of features is determined by the number of address bits. For example, 2 address bits will result in a 6 bit multiplexer and consequently 6 features (2 + 2^2 = 6). If address_bits=3 , then this results in an 11-bit multiplexer as (2 + 2^3 = 11) with 11 features. sample_size : int (default: 100) The total number of samples generated. positive_class_ratio : float (default: 0.5) The fraction (a float between 0 and 1) of samples in the sample_size d dataset that have class label 1. If positive_class_ratio=0.5 (default), then the ratio of class 0 and class 1 samples is perfectly balanced. shuffle : Bool (default: False) Whether or not to shuffle the features and labels. If False (default), the samples are returned in sorted order starting with sample_size /2 samples with class label 0 and followed by sample_size /2 samples with class label 1. random_seed : int (default: None) Random seed used for generating the multiplexer samples and shuffling. Returns X, y : [n_samples, n_features], [n_class_labels] X is the feature matrix with the number of samples equal to sample_size . The number of features is determined by the number of address bits. For instance, 2 address bits will result in a 6 bit multiplexer and consequently 6 features (2 + 2^2 = 6). All features are binary (values in {0, 1}). y is a 1-dimensional array of class labels in {0, 1}. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/data/make_multiplexer_dataset","title":"Make multiplexer dataset"},{"location":"api_modules/mlxtend.data/make_multiplexer_dataset/#make_multiplexer_dataset","text":"make_multiplexer_dataset(address_bits=2, sample_size=100, positive_class_ratio=0.5, shuffle=False, random_seed=None) Function to create a binary n-bit multiplexer dataset. New in mlxtend v0.9 Parameters address_bits : int (default: 2) A positive integer that determines the number of address bits in the multiplexer, which in turn determine the n-bit capacity of the multiplexer and therefore the number of features. The number of features is determined by the number of address bits. For example, 2 address bits will result in a 6 bit multiplexer and consequently 6 features (2 + 2^2 = 6). If address_bits=3 , then this results in an 11-bit multiplexer as (2 + 2^3 = 11) with 11 features. sample_size : int (default: 100) The total number of samples generated. positive_class_ratio : float (default: 0.5) The fraction (a float between 0 and 1) of samples in the sample_size d dataset that have class label 1. If positive_class_ratio=0.5 (default), then the ratio of class 0 and class 1 samples is perfectly balanced. shuffle : Bool (default: False) Whether or not to shuffle the features and labels. If False (default), the samples are returned in sorted order starting with sample_size /2 samples with class label 0 and followed by sample_size /2 samples with class label 1. random_seed : int (default: None) Random seed used for generating the multiplexer samples and shuffling. Returns X, y : [n_samples, n_features], [n_class_labels] X is the feature matrix with the number of samples equal to sample_size . The number of features is determined by the number of address bits. For instance, 2 address bits will result in a 6 bit multiplexer and consequently 6 features (2 + 2^2 = 6). All features are binary (values in {0, 1}). y is a 1-dimensional array of class labels in {0, 1}. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/data/make_multiplexer_dataset","title":"make_multiplexer_dataset"},{"location":"api_modules/mlxtend.data/mnist_data/","text":"mnist_data mnist_data() 5000 samples from the MNIST handwritten digits dataset. Data Source : https://yann.lecun.com/exdb/mnist/ Returns X, y : [n_samples, n_features], [n_class_labels] X is the feature matrix with 5000 image samples as rows, each row consists of 28x28 pixels that were unrolled into 784 pixel feature vectors. y contains the 10 unique class labels 0-9. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/data/mnist_data/","title":"Mnist data"},{"location":"api_modules/mlxtend.data/mnist_data/#mnist_data","text":"mnist_data() 5000 samples from the MNIST handwritten digits dataset. Data Source : https://yann.lecun.com/exdb/mnist/ Returns X, y : [n_samples, n_features], [n_class_labels] X is the feature matrix with 5000 image samples as rows, each row consists of 28x28 pixels that were unrolled into 784 pixel feature vectors. y contains the 10 unique class labels 0-9. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/data/mnist_data/","title":"mnist_data"},{"location":"api_modules/mlxtend.data/three_blobs_data/","text":"three_blobs_data three_blobs_data() A random dataset of 3 2D blobs for clustering. Number of samples : 150 Suggested labels : {0, 1, 2}, distribution: [50, 50, 50] Returns X, y : [n_samples, n_features], [n_cluster_labels] X is the feature matrix with 159 samples as rows and 2 feature columns. y is a 1-dimensional array of the 3 suggested cluster labels 0, 1, 2 Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/data/three_blobs_data","title":"Three blobs data"},{"location":"api_modules/mlxtend.data/three_blobs_data/#three_blobs_data","text":"three_blobs_data() A random dataset of 3 2D blobs for clustering. Number of samples : 150 Suggested labels : {0, 1, 2}, distribution: [50, 50, 50] Returns X, y : [n_samples, n_features], [n_cluster_labels] X is the feature matrix with 159 samples as rows and 2 feature columns. y is a 1-dimensional array of the 3 suggested cluster labels 0, 1, 2 Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/data/three_blobs_data","title":"three_blobs_data"},{"location":"api_modules/mlxtend.data/wine_data/","text":"wine_data wine_data() Wine dataset. Source : https://archive.ics.uci.edu/ml/datasets/Wine Number of samples : 178 Class labels : {0, 1, 2}, distribution: [59, 71, 48] Dataset Attributes: 1) Alcohol 2) Malic acid 3) Ash 4) Alcalinity of ash 5) Magnesium 6) Total phenols 7) Flavanoids 8) Nonflavanoid phenols 9) Proanthocyanins 10) Color intensity 11) Hue 12) OD280/OD315 of diluted wines 13) Proline Returns X, y : [n_samples, n_features], [n_class_labels] X is the feature matrix with 178 wine samples as rows and 13 feature columns. y is a 1-dimensional array of the 3 class labels 0, 1, 2 Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/data/wine_data","title":"Wine data"},{"location":"api_modules/mlxtend.data/wine_data/#wine_data","text":"wine_data() Wine dataset. Source : https://archive.ics.uci.edu/ml/datasets/Wine Number of samples : 178 Class labels : {0, 1, 2}, distribution: [59, 71, 48] Dataset Attributes: 1) Alcohol 2) Malic acid 3) Ash 4) Alcalinity of ash 5) Magnesium 6) Total phenols 7) Flavanoids 8) Nonflavanoid phenols 9) Proanthocyanins 10) Color intensity 11) Hue 12) OD280/OD315 of diluted wines 13) Proline Returns X, y : [n_samples, n_features], [n_class_labels] X is the feature matrix with 178 wine samples as rows and 13 feature columns. y is a 1-dimensional array of the 3 class labels 0, 1, 2 Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/data/wine_data","title":"wine_data"},{"location":"api_modules/mlxtend.evaluate/BootstrapOutOfBag/","text":"BootstrapOutOfBag BootstrapOutOfBag(n_splits=200, random_seed=None) Parameters n_splits : int (default=200) Number of bootstrap iterations. Must be larger than 1. random_seed : int (default=None) If int, random_seed is the seed used by the random number generator. Returns train_idx : ndarray The training set indices for that split. test_idx : ndarray The testing set indices for that split. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/BootstrapOutOfBag/ Methods get_n_splits(X=None, y=None, groups=None) Returns the number of splitting iterations in the cross-validator Parameters X : object Always ignored, exists for compatibility with scikit-learn. y : object Always ignored, exists for compatibility with scikit-learn. groups : object Always ignored, exists for compatibility with scikit-learn. Returns n_splits : int Returns the number of splitting iterations in the cross-validator. split(X, y=None, groups=None) y : array-like or None (default: None) Argument is not used and only included as parameter for compatibility, similar to KFold in scikit-learn. groups : array-like or None (default: None) Argument is not used and only included as parameter for compatibility, similar to KFold in scikit-learn.","title":"BootstrapOutOfBag"},{"location":"api_modules/mlxtend.evaluate/BootstrapOutOfBag/#bootstrapoutofbag","text":"BootstrapOutOfBag(n_splits=200, random_seed=None) Parameters n_splits : int (default=200) Number of bootstrap iterations. Must be larger than 1. random_seed : int (default=None) If int, random_seed is the seed used by the random number generator. Returns train_idx : ndarray The training set indices for that split. test_idx : ndarray The testing set indices for that split. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/BootstrapOutOfBag/","title":"BootstrapOutOfBag"},{"location":"api_modules/mlxtend.evaluate/BootstrapOutOfBag/#methods","text":"get_n_splits(X=None, y=None, groups=None) Returns the number of splitting iterations in the cross-validator Parameters X : object Always ignored, exists for compatibility with scikit-learn. y : object Always ignored, exists for compatibility with scikit-learn. groups : object Always ignored, exists for compatibility with scikit-learn. Returns n_splits : int Returns the number of splitting iterations in the cross-validator. split(X, y=None, groups=None) y : array-like or None (default: None) Argument is not used and only included as parameter for compatibility, similar to KFold in scikit-learn. groups : array-like or None (default: None) Argument is not used and only included as parameter for compatibility, similar to KFold in scikit-learn.","title":"Methods"},{"location":"api_modules/mlxtend.evaluate/GroupTimeSeriesSplit/","text":"GroupTimeSeriesSplit GroupTimeSeriesSplit(test_size, train_size=None, n_splits=None, gap_size=0, shift_size=1, window_type='rolling') Group time series cross-validator. Parameters test_size : int Size of test dataset. train_size : int (default=None) Size of train dataset. n_splits : int (default=None) Number of the splits. gap_size : int (default=0) Gap size between train and test datasets. shift_size : int (default=1) Step to shift for the next fold. window_type : str (default=\"rolling\") Type of the window. Possible values: \"rolling\", \"expanding\". Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/GroupTimeSeriesSplit/ Methods get_n_splits(X=None, y=None, groups=None) Returns the number of splitting iterations in the cross-validator. Parameters X : object Always ignored, exists for compatibility. y : object Always ignored, exists for compatibility. groups : object Always ignored, exists for compatibility. Returns n_splits : int Returns the number of splitting iterations in the cross-validator. split(X, y=None, groups=None) Generate indices to split data into training and test set. Parameters X : array-like Training data. y : array-like (default=None) Always ignored, exists for compatibility. groups : array-like (default=None) Array with group names or sequence numbers. Yields train : ndarray The training set indices for that split. test : ndarray The testing set indices for that split.","title":"GroupTimeSeriesSplit"},{"location":"api_modules/mlxtend.evaluate/GroupTimeSeriesSplit/#grouptimeseriessplit","text":"GroupTimeSeriesSplit(test_size, train_size=None, n_splits=None, gap_size=0, shift_size=1, window_type='rolling') Group time series cross-validator. Parameters test_size : int Size of test dataset. train_size : int (default=None) Size of train dataset. n_splits : int (default=None) Number of the splits. gap_size : int (default=0) Gap size between train and test datasets. shift_size : int (default=1) Step to shift for the next fold. window_type : str (default=\"rolling\") Type of the window. Possible values: \"rolling\", \"expanding\". Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/GroupTimeSeriesSplit/","title":"GroupTimeSeriesSplit"},{"location":"api_modules/mlxtend.evaluate/GroupTimeSeriesSplit/#methods","text":"get_n_splits(X=None, y=None, groups=None) Returns the number of splitting iterations in the cross-validator. Parameters X : object Always ignored, exists for compatibility. y : object Always ignored, exists for compatibility. groups : object Always ignored, exists for compatibility. Returns n_splits : int Returns the number of splitting iterations in the cross-validator. split(X, y=None, groups=None) Generate indices to split data into training and test set. Parameters X : array-like Training data. y : array-like (default=None) Always ignored, exists for compatibility. groups : array-like (default=None) Array with group names or sequence numbers. Yields train : ndarray The training set indices for that split. test : ndarray The testing set indices for that split.","title":"Methods"},{"location":"api_modules/mlxtend.evaluate/PredefinedHoldoutSplit/","text":"PredefinedHoldoutSplit PredefinedHoldoutSplit(valid_indices) Train/Validation set splitter for sklearn's GridSearchCV etc. Uses user-specified train/validation set indices to split a dataset into train/validation sets using user-defined or random indices. Parameters valid_indices : array-like, shape (num_examples,) Indices of the training examples in the training set to be used for validation. All other indices in the training set are used to for a training subset for model fitting. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/PredefinedHoldoutSplit/ Methods get_n_splits(X=None, y=None, groups=None) Returns the number of splitting iterations in the cross-validator Parameters X : object Always ignored, exists for compatibility. y : object Always ignored, exists for compatibility. groups : object Always ignored, exists for compatibility. Returns n_splits : 1 Returns the number of splitting iterations in the cross-validator. Always returns 1. split(X, y, groups=None) Generate indices to split data into training and test set. Parameters X : array-like, shape (num_examples, num_features) Training data, where num_examples is the number of examples and num_features is the number of features. y : array-like, shape (num_examples,) The target variable for supervised learning problems. Stratification is done based on the y labels. groups : object Always ignored, exists for compatibility. Yields train_index : ndarray The training set indices for that split. valid_index : ndarray The validation set indices for that split.","title":"PredefinedHoldoutSplit"},{"location":"api_modules/mlxtend.evaluate/PredefinedHoldoutSplit/#predefinedholdoutsplit","text":"PredefinedHoldoutSplit(valid_indices) Train/Validation set splitter for sklearn's GridSearchCV etc. Uses user-specified train/validation set indices to split a dataset into train/validation sets using user-defined or random indices. Parameters valid_indices : array-like, shape (num_examples,) Indices of the training examples in the training set to be used for validation. All other indices in the training set are used to for a training subset for model fitting. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/PredefinedHoldoutSplit/","title":"PredefinedHoldoutSplit"},{"location":"api_modules/mlxtend.evaluate/PredefinedHoldoutSplit/#methods","text":"get_n_splits(X=None, y=None, groups=None) Returns the number of splitting iterations in the cross-validator Parameters X : object Always ignored, exists for compatibility. y : object Always ignored, exists for compatibility. groups : object Always ignored, exists for compatibility. Returns n_splits : 1 Returns the number of splitting iterations in the cross-validator. Always returns 1. split(X, y, groups=None) Generate indices to split data into training and test set. Parameters X : array-like, shape (num_examples, num_features) Training data, where num_examples is the number of examples and num_features is the number of features. y : array-like, shape (num_examples,) The target variable for supervised learning problems. Stratification is done based on the y labels. groups : object Always ignored, exists for compatibility. Yields train_index : ndarray The training set indices for that split. valid_index : ndarray The validation set indices for that split.","title":"Methods"},{"location":"api_modules/mlxtend.evaluate/RandomHoldoutSplit/","text":"RandomHoldoutSplit RandomHoldoutSplit(valid_size=0.5, random_seed=None, stratify=False) Train/Validation set splitter for sklearn's GridSearchCV etc. Provides train/validation set indices to split a dataset into train/validation sets using random indices. Parameters valid_size : float (default: 0.5) Proportion of examples that being assigned as validation examples. 1- valid_size will then automatically be assigned as training set examples. random_seed : int (default: None) The random seed for splitting the data into training and validation set partitions. stratify : bool (default: False) True or False, whether to perform a stratified split or not Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/RandomHoldoutSplit/ Methods get_n_splits(X=None, y=None, groups=None) Returns the number of splitting iterations in the cross-validator Parameters X : object Always ignored, exists for compatibility. y : object Always ignored, exists for compatibility. groups : object Always ignored, exists for compatibility. Returns n_splits : 1 Returns the number of splitting iterations in the cross-validator. Always returns 1. split(X, y, groups=None) Generate indices to split data into training and test set. Parameters X : array-like, shape (num_examples, num_features) Training data, where num_examples is the number of training examples and num_features is the number of features. y : array-like, shape (num_examples,) The target variable for supervised learning problems. Stratification is done based on the y labels. groups : object Always ignored, exists for compatibility. Yields train_index : ndarray The training set indices for that split. valid_index : ndarray The validation set indices for that split.","title":"RandomHoldoutSplit"},{"location":"api_modules/mlxtend.evaluate/RandomHoldoutSplit/#randomholdoutsplit","text":"RandomHoldoutSplit(valid_size=0.5, random_seed=None, stratify=False) Train/Validation set splitter for sklearn's GridSearchCV etc. Provides train/validation set indices to split a dataset into train/validation sets using random indices. Parameters valid_size : float (default: 0.5) Proportion of examples that being assigned as validation examples. 1- valid_size will then automatically be assigned as training set examples. random_seed : int (default: None) The random seed for splitting the data into training and validation set partitions. stratify : bool (default: False) True or False, whether to perform a stratified split or not Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/RandomHoldoutSplit/","title":"RandomHoldoutSplit"},{"location":"api_modules/mlxtend.evaluate/RandomHoldoutSplit/#methods","text":"get_n_splits(X=None, y=None, groups=None) Returns the number of splitting iterations in the cross-validator Parameters X : object Always ignored, exists for compatibility. y : object Always ignored, exists for compatibility. groups : object Always ignored, exists for compatibility. Returns n_splits : 1 Returns the number of splitting iterations in the cross-validator. Always returns 1. split(X, y, groups=None) Generate indices to split data into training and test set. Parameters X : array-like, shape (num_examples, num_features) Training data, where num_examples is the number of training examples and num_features is the number of features. y : array-like, shape (num_examples,) The target variable for supervised learning problems. Stratification is done based on the y labels. groups : object Always ignored, exists for compatibility. Yields train_index : ndarray The training set indices for that split. valid_index : ndarray The validation set indices for that split.","title":"Methods"},{"location":"api_modules/mlxtend.evaluate/accuracy_score/","text":"accuracy_score accuracy_score(y_target, y_predicted, method='standard', pos_label=1, normalize=True) General accuracy function for supervised learning. Parameters y_target : array-like, shape=[n_values] True class labels or target values. y_predicted : array-like, shape=[n_values] Predicted class labels or target values. method : str, 'standard' by default. The chosen method for accuracy computation. If set to 'standard', computes overall accuracy. If set to 'binary', computes accuracy for class pos_label. If set to 'average', computes average per-class (balanced) accuracy. If set to 'balanced', computes the scikit-learn-style balanced accuracy. pos_label : str or int, 1 by default. The class whose accuracy score is to be reported. Used only when method is set to 'binary' normalize : bool, True by default. If True, returns fraction of correctly classified samples. If False, returns number of correctly classified samples. Returns score: float Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/accuracy_score/","title":"Accuracy score"},{"location":"api_modules/mlxtend.evaluate/accuracy_score/#accuracy_score","text":"accuracy_score(y_target, y_predicted, method='standard', pos_label=1, normalize=True) General accuracy function for supervised learning. Parameters y_target : array-like, shape=[n_values] True class labels or target values. y_predicted : array-like, shape=[n_values] Predicted class labels or target values. method : str, 'standard' by default. The chosen method for accuracy computation. If set to 'standard', computes overall accuracy. If set to 'binary', computes accuracy for class pos_label. If set to 'average', computes average per-class (balanced) accuracy. If set to 'balanced', computes the scikit-learn-style balanced accuracy. pos_label : str or int, 1 by default. The class whose accuracy score is to be reported. Used only when method is set to 'binary' normalize : bool, True by default. If True, returns fraction of correctly classified samples. If False, returns number of correctly classified samples. Returns score: float Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/accuracy_score/","title":"accuracy_score"},{"location":"api_modules/mlxtend.evaluate/bias_variance_decomp/","text":"bias_variance_decomp bias_variance_decomp(estimator, X_train, y_train, X_test, y_test, loss='0-1_loss', num_rounds=200, random_seed=None, fit_params) estimator : object A classifier or regressor object or class implementing both a fit and predict method similar to the scikit-learn API. X_train : array-like, shape=(num_examples, num_features) A training dataset for drawing the bootstrap samples to carry out the bias-variance decomposition. y_train : array-like, shape=(num_examples) Targets (class labels, continuous values in case of regression) associated with the X_train examples. X_test : array-like, shape=(num_examples, num_features) The test dataset for computing the average loss, bias, and variance. y_test : array-like, shape=(num_examples) Targets (class labels, continuous values in case of regression) associated with the X_test examples. loss : str (default='0-1_loss') Loss function for performing the bias-variance decomposition. Currently allowed values are '0-1_loss' and 'mse'. num_rounds : int (default=200) Number of bootstrap rounds (sampling from the training set) for performing the bias-variance decomposition. Each bootstrap sample has the same size as the original training set. random_seed : int (default=None) Random seed for the bootstrap sampling used for the bias-variance decomposition. fit_params : additional parameters Additional parameters to be passed to the .fit() function of the estimator when it is fit to the bootstrap samples. Returns avg_expected_loss, avg_bias, avg_var : returns the average expected average bias, and average bias (all floats), where the average is computed over the data points in the test set. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/bias_variance_decomp/","title":"Bias variance decomp"},{"location":"api_modules/mlxtend.evaluate/bias_variance_decomp/#bias_variance_decomp","text":"bias_variance_decomp(estimator, X_train, y_train, X_test, y_test, loss='0-1_loss', num_rounds=200, random_seed=None, fit_params) estimator : object A classifier or regressor object or class implementing both a fit and predict method similar to the scikit-learn API. X_train : array-like, shape=(num_examples, num_features) A training dataset for drawing the bootstrap samples to carry out the bias-variance decomposition. y_train : array-like, shape=(num_examples) Targets (class labels, continuous values in case of regression) associated with the X_train examples. X_test : array-like, shape=(num_examples, num_features) The test dataset for computing the average loss, bias, and variance. y_test : array-like, shape=(num_examples) Targets (class labels, continuous values in case of regression) associated with the X_test examples. loss : str (default='0-1_loss') Loss function for performing the bias-variance decomposition. Currently allowed values are '0-1_loss' and 'mse'. num_rounds : int (default=200) Number of bootstrap rounds (sampling from the training set) for performing the bias-variance decomposition. Each bootstrap sample has the same size as the original training set. random_seed : int (default=None) Random seed for the bootstrap sampling used for the bias-variance decomposition. fit_params : additional parameters Additional parameters to be passed to the .fit() function of the estimator when it is fit to the bootstrap samples. Returns avg_expected_loss, avg_bias, avg_var : returns the average expected average bias, and average bias (all floats), where the average is computed over the data points in the test set. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/bias_variance_decomp/","title":"bias_variance_decomp"},{"location":"api_modules/mlxtend.evaluate/bootstrap/","text":"bootstrap bootstrap(x, func, num_rounds=1000, ci=0.95, ddof=1, seed=None) Implements the ordinary nonparametric bootstrap Parameters x : NumPy array, shape=(n_samples, [n_columns]) An one or multidimensional array of data records func : A function which computes a statistic that is used to compute the bootstrap replicates (the statistic computed from the bootstrap samples). This function must return a scalar value. For example, np.mean or np.median would be an acceptable argument for func if x is a 1-dimensional array or vector. num_rounds : int (default=1000) The number of bootstrap samples to draw where each bootstrap sample has the same number of records as the original dataset. ci : int (default=0.95) An integer in the range (0, 1) that represents the confidence level for computing the confidence interval. For example, ci=0.95 (default) will compute the 95% confidence interval from the bootstrap replicates. ddof : int The delta degrees of freedom used when computing the standard error. seed : int or None (default=None) Random seed for generating bootstrap samples. Returns original, standard_error, (lower_ci, upper_ci) : tuple Returns the statistic of the original sample ( original ), the standard error of the estimate, and the respective confidence interval bounds. Examples ``` >>> from mlxtend.evaluate import bootstrap >>> rng = np.random.RandomState(123) >>> x = rng.normal(loc=5., size=100) >>> original, std_err, ci_bounds = bootstrap(x, ... num_rounds=1000, ... func=np.mean, ... ci=0.95, ... seed=123) >>> print('Mean: %.2f, SE: +/- %.2f, CI95: [%.2f, %.2f]' % (original, ... std_err, ... ci_bounds[0], ... ci_bounds[1])) Mean: 5.03, SE: +/- 0.11, CI95: [4.80, 5.26] >>> For more usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/bootstrap/ ```","title":"Bootstrap"},{"location":"api_modules/mlxtend.evaluate/bootstrap/#bootstrap","text":"bootstrap(x, func, num_rounds=1000, ci=0.95, ddof=1, seed=None) Implements the ordinary nonparametric bootstrap Parameters x : NumPy array, shape=(n_samples, [n_columns]) An one or multidimensional array of data records func : A function which computes a statistic that is used to compute the bootstrap replicates (the statistic computed from the bootstrap samples). This function must return a scalar value. For example, np.mean or np.median would be an acceptable argument for func if x is a 1-dimensional array or vector. num_rounds : int (default=1000) The number of bootstrap samples to draw where each bootstrap sample has the same number of records as the original dataset. ci : int (default=0.95) An integer in the range (0, 1) that represents the confidence level for computing the confidence interval. For example, ci=0.95 (default) will compute the 95% confidence interval from the bootstrap replicates. ddof : int The delta degrees of freedom used when computing the standard error. seed : int or None (default=None) Random seed for generating bootstrap samples. Returns original, standard_error, (lower_ci, upper_ci) : tuple Returns the statistic of the original sample ( original ), the standard error of the estimate, and the respective confidence interval bounds. Examples ``` >>> from mlxtend.evaluate import bootstrap >>> rng = np.random.RandomState(123) >>> x = rng.normal(loc=5., size=100) >>> original, std_err, ci_bounds = bootstrap(x, ... num_rounds=1000, ... func=np.mean, ... ci=0.95, ... seed=123) >>> print('Mean: %.2f, SE: +/- %.2f, CI95: [%.2f, %.2f]' % (original, ... std_err, ... ci_bounds[0], ... ci_bounds[1])) Mean: 5.03, SE: +/- 0.11, CI95: [4.80, 5.26] >>> For more usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/bootstrap/ ```","title":"bootstrap"},{"location":"api_modules/mlxtend.evaluate/bootstrap_point632_score/","text":"bootstrap_point632_score bootstrap_point632_score(estimator, X, y, n_splits=200, method='.632', scoring_func=None, predict_proba=False, random_seed=None, clone_estimator=True, fit_params) Implementation of the .632 [1] and .632+ [2] bootstrap for supervised learning References: - [1] Efron, Bradley. 1983. \"Estimating the Error Rate of a Prediction Rule: Improvement on Cross-Validation.\" Journal of the American Statistical Association 78 (382): 316. doi:10.2307/2288636. - [2] Efron, Bradley, and Robert Tibshirani. 1997. \"Improvements on Cross-Validation: The .632+ Bootstrap Method.\" Journal of the American Statistical Association 92 (438): 548. doi:10.2307/2965703. Parameters estimator : object An estimator for classification or regression that follows the scikit-learn API and implements \"fit\" and \"predict\" methods. X : array-like The data to fit. Can be, for example a list, or an array at least 2d. y : array-like, optional, default: None The target variable to try to predict in the case of supervised learning. n_splits : int (default=200) Number of bootstrap iterations. Must be larger than 1. method : str (default='.632') The bootstrap method, which can be either - 1) '.632' bootstrap (default) - 2) '.632+' bootstrap - 3) 'oob' (regular out-of-bag, no weighting) for comparison studies. scoring_func : callable, Score function (or loss function) with signature scoring_func(y, y_pred, **kwargs) . If none, uses classification accuracy if the estimator is a classifier and mean squared error if the estimator is a regressor. predict_proba : bool Whether to use the predict_proba function for the estimator argument. This is to be used in conjunction with scoring_func which takes in probability values instead of actual predictions. For example, if the scoring_func is :meth: sklearn.metrics.roc_auc_score , then use predict_proba=True . Note that this requires estimator to have predict_proba method implemented. random_seed : int (default=None) If int, random_seed is the seed used by the random number generator. clone_estimator : bool (default=True) Clones the estimator if true, otherwise fits the original. fit_params : additional parameters Additional parameters to be passed to the .fit() function of the estimator when it is fit to the bootstrap samples. Returns scores : array of float, shape=(len(list(n_splits)),) Array of scores of the estimator for each bootstrap replicate. Examples >>> from sklearn import datasets, linear_model >>> from mlxtend.evaluate import bootstrap_point632_score >>> iris = datasets.load_iris() >>> X = iris.data >>> y = iris.target >>> lr = linear_model.LogisticRegression() >>> scores = bootstrap_point632_score(lr, X, y) >>> acc = np.mean(scores) >>> print('Accuracy:', acc) 0.953023146884 >>> lower = np.percentile(scores, 2.5) >>> upper = np.percentile(scores, 97.5) >>> print('95%% Confidence interval: [%.2f, %.2f]' % (lower, upper)) 95% Confidence interval: [0.90, 0.98] For more usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/bootstrap_point632_score/","title":"Bootstrap point632 score"},{"location":"api_modules/mlxtend.evaluate/bootstrap_point632_score/#bootstrap_point632_score","text":"bootstrap_point632_score(estimator, X, y, n_splits=200, method='.632', scoring_func=None, predict_proba=False, random_seed=None, clone_estimator=True, fit_params) Implementation of the .632 [1] and .632+ [2] bootstrap for supervised learning References: - [1] Efron, Bradley. 1983. \"Estimating the Error Rate of a Prediction Rule: Improvement on Cross-Validation.\" Journal of the American Statistical Association 78 (382): 316. doi:10.2307/2288636. - [2] Efron, Bradley, and Robert Tibshirani. 1997. \"Improvements on Cross-Validation: The .632+ Bootstrap Method.\" Journal of the American Statistical Association 92 (438): 548. doi:10.2307/2965703. Parameters estimator : object An estimator for classification or regression that follows the scikit-learn API and implements \"fit\" and \"predict\" methods. X : array-like The data to fit. Can be, for example a list, or an array at least 2d. y : array-like, optional, default: None The target variable to try to predict in the case of supervised learning. n_splits : int (default=200) Number of bootstrap iterations. Must be larger than 1. method : str (default='.632') The bootstrap method, which can be either - 1) '.632' bootstrap (default) - 2) '.632+' bootstrap - 3) 'oob' (regular out-of-bag, no weighting) for comparison studies. scoring_func : callable, Score function (or loss function) with signature scoring_func(y, y_pred, **kwargs) . If none, uses classification accuracy if the estimator is a classifier and mean squared error if the estimator is a regressor. predict_proba : bool Whether to use the predict_proba function for the estimator argument. This is to be used in conjunction with scoring_func which takes in probability values instead of actual predictions. For example, if the scoring_func is :meth: sklearn.metrics.roc_auc_score , then use predict_proba=True . Note that this requires estimator to have predict_proba method implemented. random_seed : int (default=None) If int, random_seed is the seed used by the random number generator. clone_estimator : bool (default=True) Clones the estimator if true, otherwise fits the original. fit_params : additional parameters Additional parameters to be passed to the .fit() function of the estimator when it is fit to the bootstrap samples. Returns scores : array of float, shape=(len(list(n_splits)),) Array of scores of the estimator for each bootstrap replicate. Examples >>> from sklearn import datasets, linear_model >>> from mlxtend.evaluate import bootstrap_point632_score >>> iris = datasets.load_iris() >>> X = iris.data >>> y = iris.target >>> lr = linear_model.LogisticRegression() >>> scores = bootstrap_point632_score(lr, X, y) >>> acc = np.mean(scores) >>> print('Accuracy:', acc) 0.953023146884 >>> lower = np.percentile(scores, 2.5) >>> upper = np.percentile(scores, 97.5) >>> print('95%% Confidence interval: [%.2f, %.2f]' % (lower, upper)) 95% Confidence interval: [0.90, 0.98] For more usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/bootstrap_point632_score/","title":"bootstrap_point632_score"},{"location":"api_modules/mlxtend.evaluate/cochrans_q/","text":"cochrans_q cochrans_q(y_target, y_model_predictions)* Cochran's Q test to compare 2 or more models. Parameters y_target : array-like, shape=[n_samples] True class labels as 1D NumPy array. *y_model_predictions : array-likes, shape=[n_samples] Variable number of 2 or more arrays that contain the predicted class labels from models as 1D NumPy array. Returns q, p : float or None, float Returns the Q (chi-squared) value and the p-value Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/cochrans_q/","title":"Cochrans q"},{"location":"api_modules/mlxtend.evaluate/cochrans_q/#cochrans_q","text":"cochrans_q(y_target, y_model_predictions)* Cochran's Q test to compare 2 or more models. Parameters y_target : array-like, shape=[n_samples] True class labels as 1D NumPy array. *y_model_predictions : array-likes, shape=[n_samples] Variable number of 2 or more arrays that contain the predicted class labels from models as 1D NumPy array. Returns q, p : float or None, float Returns the Q (chi-squared) value and the p-value Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/cochrans_q/","title":"cochrans_q"},{"location":"api_modules/mlxtend.evaluate/combined_ftest_5x2cv/","text":"combined_ftest_5x2cv combined_ftest_5x2cv(estimator1, estimator2, X, y, scoring=None, random_seed=None) Implements the 5x2cv combined F test proposed by Alpaydin 1999, to compare the performance of two models. Parameters estimator1 : scikit-learn classifier or regressor estimator2 : scikit-learn classifier or regressor X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values. scoring : str, callable, or None (default: None) If None (default), uses 'accuracy' for sklearn classifiers and 'r2' for sklearn regressors. If str, uses a sklearn scoring metric string identifier, for example {accuracy, f1, precision, recall, roc_auc} for classifiers, {'mean_absolute_error', 'mean_squared_error'/'neg_mean_squared_error', 'median_absolute_error', 'r2'} for regressors. If a callable object or function is provided, it has to be conform with sklearn's signature scorer(estimator, X, y) ; see https://scikit-learn.org/stable/modules/generated/sklearn.metrics.make_scorer.html for more information. random_seed : int or None (default: None) Random seed for creating the test/train splits. Returns f : float The F-statistic pvalue : float Two-tailed p-value. If the chosen significance level is larger than the p-value, we reject the null hypothesis and accept that there are significant differences in the two compared models. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/combined_ftest_5x2cv/","title":"Combined ftest 5x2cv"},{"location":"api_modules/mlxtend.evaluate/combined_ftest_5x2cv/#combined_ftest_5x2cv","text":"combined_ftest_5x2cv(estimator1, estimator2, X, y, scoring=None, random_seed=None) Implements the 5x2cv combined F test proposed by Alpaydin 1999, to compare the performance of two models. Parameters estimator1 : scikit-learn classifier or regressor estimator2 : scikit-learn classifier or regressor X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values. scoring : str, callable, or None (default: None) If None (default), uses 'accuracy' for sklearn classifiers and 'r2' for sklearn regressors. If str, uses a sklearn scoring metric string identifier, for example {accuracy, f1, precision, recall, roc_auc} for classifiers, {'mean_absolute_error', 'mean_squared_error'/'neg_mean_squared_error', 'median_absolute_error', 'r2'} for regressors. If a callable object or function is provided, it has to be conform with sklearn's signature scorer(estimator, X, y) ; see https://scikit-learn.org/stable/modules/generated/sklearn.metrics.make_scorer.html for more information. random_seed : int or None (default: None) Random seed for creating the test/train splits. Returns f : float The F-statistic pvalue : float Two-tailed p-value. If the chosen significance level is larger than the p-value, we reject the null hypothesis and accept that there are significant differences in the two compared models. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/combined_ftest_5x2cv/","title":"combined_ftest_5x2cv"},{"location":"api_modules/mlxtend.evaluate/confusion_matrix/","text":"confusion_matrix confusion_matrix(y_target, y_predicted, binary=False, positive_label=1) Compute a confusion matrix/contingency table. Parameters y_target : array-like, shape=[n_samples] True class labels. y_predicted : array-like, shape=[n_samples] Predicted class labels. binary : bool (default: False) Maps a multi-class problem onto a binary confusion matrix, where the positive class is 1 and all other classes are 0. positive_label : int (default: 1) Class label of the positive class. Returns mat : array-like, shape=[n_classes, n_classes] Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/confusion_matrix/","title":"Confusion matrix"},{"location":"api_modules/mlxtend.evaluate/confusion_matrix/#confusion_matrix","text":"confusion_matrix(y_target, y_predicted, binary=False, positive_label=1) Compute a confusion matrix/contingency table. Parameters y_target : array-like, shape=[n_samples] True class labels. y_predicted : array-like, shape=[n_samples] Predicted class labels. binary : bool (default: False) Maps a multi-class problem onto a binary confusion matrix, where the positive class is 1 and all other classes are 0. positive_label : int (default: 1) Class label of the positive class. Returns mat : array-like, shape=[n_classes, n_classes] Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/confusion_matrix/","title":"confusion_matrix"},{"location":"api_modules/mlxtend.evaluate/create_counterfactual/","text":"create_counterfactual create_counterfactual(x_reference, y_desired, model, X_dataset, y_desired_proba=None, lammbda=0.1, random_seed=None) Implementation of the counterfactual method by Wachter et al. 2017 References: - Wachter, S., Mittelstadt, B., & Russell, C. (2017). Counterfactual explanations without opening the black box: Automated decisions and the GDPR. Harv. JL & Tech., 31, 841., https://arxiv.org/abs/1711.00399 Parameters x_reference : array-like, shape=[m_features] The data instance (training example) to be explained. y_desired : int The desired class label for x_reference . model : estimator A (scikit-learn) estimator implementing .predict() and/or predict_proba() . - If model supports predict_proba() , then this is used by default for the first loss term, (lambda * model.predict[_proba](x_counterfact) - y_desired[_proba])^2 - Otherwise, method will fall back to predict . X_dataset : array-like, shape=[n_examples, m_features] A (training) dataset for picking the initial counterfactual as initial value for starting the optimization procedure. y_desired_proba : float (default: None) A float within the range [0, 1] designating the desired class probability for y_desired . - If y_desired_proba=None (default), the first loss term is (lambda * model(x_counterfact) - y_desired)^2 where y_desired is a class label - If y_desired_proba is not None, the first loss term is (lambda * model(x_counterfact) - y_desired_proba)^2 lammbda : Weighting parameter for the first loss term, (lambda * model(x_counterfact) - y_desired[_proba])^2 random_seed : int (default=None) If int, random_seed is the seed used by the random number generator for selecting the inital counterfactual from X_dataset .","title":"Create counterfactual"},{"location":"api_modules/mlxtend.evaluate/create_counterfactual/#create_counterfactual","text":"create_counterfactual(x_reference, y_desired, model, X_dataset, y_desired_proba=None, lammbda=0.1, random_seed=None) Implementation of the counterfactual method by Wachter et al. 2017 References: - Wachter, S., Mittelstadt, B., & Russell, C. (2017). Counterfactual explanations without opening the black box: Automated decisions and the GDPR. Harv. JL & Tech., 31, 841., https://arxiv.org/abs/1711.00399 Parameters x_reference : array-like, shape=[m_features] The data instance (training example) to be explained. y_desired : int The desired class label for x_reference . model : estimator A (scikit-learn) estimator implementing .predict() and/or predict_proba() . - If model supports predict_proba() , then this is used by default for the first loss term, (lambda * model.predict[_proba](x_counterfact) - y_desired[_proba])^2 - Otherwise, method will fall back to predict . X_dataset : array-like, shape=[n_examples, m_features] A (training) dataset for picking the initial counterfactual as initial value for starting the optimization procedure. y_desired_proba : float (default: None) A float within the range [0, 1] designating the desired class probability for y_desired . - If y_desired_proba=None (default), the first loss term is (lambda * model(x_counterfact) - y_desired)^2 where y_desired is a class label - If y_desired_proba is not None, the first loss term is (lambda * model(x_counterfact) - y_desired_proba)^2 lammbda : Weighting parameter for the first loss term, (lambda * model(x_counterfact) - y_desired[_proba])^2 random_seed : int (default=None) If int, random_seed is the seed used by the random number generator for selecting the inital counterfactual from X_dataset .","title":"create_counterfactual"},{"location":"api_modules/mlxtend.evaluate/feature_importance_permutation/","text":"feature_importance_permutation feature_importance_permutation(X, y, predict_method, metric, num_rounds=1, feature_groups=None, seed=None) Feature importance imputation via permutation importance Parameters X : NumPy array, shape = [n_samples, n_features] Dataset, where n_samples is the number of samples and n_features is the number of features. y : NumPy array, shape = [n_samples] Target values. predict_method : prediction function A callable function that predicts the target values from X. metric : str, callable The metric for evaluating the feature importance through permutation. By default, the strings 'accuracy' is recommended for classifiers and the string 'r2' is recommended for regressors. Optionally, a custom scoring function (e.g., metric=scoring_func ) that accepts two arguments, y_true and y_pred, which have similar shape to the y array. num_rounds : int (default=1) Number of rounds the feature columns are permuted to compute the permutation importance. feature_groups : list or None (default=None) Optional argument for treating certain features as a group. For example [1, 2, [3, 4, 5]] , which can be useful for interpretability, for example, if features 3, 4, 5 are one-hot encoded features. seed : int or None (default=None) Random seed for permuting the feature columns. Returns mean_importance_vals, all_importance_vals : NumPy arrays. The first array, mean_importance_vals has shape [n_features, ] and contains the importance values for all features. The shape of the second array is [n_features, num_rounds] and contains the feature importance for each repetition. If num_rounds=1, it contains the same values as the first array, mean_importance_vals. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/feature_importance_permutation/","title":"Feature importance permutation"},{"location":"api_modules/mlxtend.evaluate/feature_importance_permutation/#feature_importance_permutation","text":"feature_importance_permutation(X, y, predict_method, metric, num_rounds=1, feature_groups=None, seed=None) Feature importance imputation via permutation importance Parameters X : NumPy array, shape = [n_samples, n_features] Dataset, where n_samples is the number of samples and n_features is the number of features. y : NumPy array, shape = [n_samples] Target values. predict_method : prediction function A callable function that predicts the target values from X. metric : str, callable The metric for evaluating the feature importance through permutation. By default, the strings 'accuracy' is recommended for classifiers and the string 'r2' is recommended for regressors. Optionally, a custom scoring function (e.g., metric=scoring_func ) that accepts two arguments, y_true and y_pred, which have similar shape to the y array. num_rounds : int (default=1) Number of rounds the feature columns are permuted to compute the permutation importance. feature_groups : list or None (default=None) Optional argument for treating certain features as a group. For example [1, 2, [3, 4, 5]] , which can be useful for interpretability, for example, if features 3, 4, 5 are one-hot encoded features. seed : int or None (default=None) Random seed for permuting the feature columns. Returns mean_importance_vals, all_importance_vals : NumPy arrays. The first array, mean_importance_vals has shape [n_features, ] and contains the importance values for all features. The shape of the second array is [n_features, num_rounds] and contains the feature importance for each repetition. If num_rounds=1, it contains the same values as the first array, mean_importance_vals. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/feature_importance_permutation/","title":"feature_importance_permutation"},{"location":"api_modules/mlxtend.evaluate/ftest/","text":"ftest ftest(y_target, y_model_predictions)* F-Test test to compare 2 or more models. Parameters y_target : array-like, shape=[n_samples] True class labels as 1D NumPy array. *y_model_predictions : array-likes, shape=[n_samples] Variable number of 2 or more arrays that contain the predicted class labels from models as 1D NumPy array. Returns f, p : float or None, float Returns the F-value and the p-value Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/ftest/","title":"Ftest"},{"location":"api_modules/mlxtend.evaluate/ftest/#ftest","text":"ftest(y_target, y_model_predictions)* F-Test test to compare 2 or more models. Parameters y_target : array-like, shape=[n_samples] True class labels as 1D NumPy array. *y_model_predictions : array-likes, shape=[n_samples] Variable number of 2 or more arrays that contain the predicted class labels from models as 1D NumPy array. Returns f, p : float or None, float Returns the F-value and the p-value Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/ftest/","title":"ftest"},{"location":"api_modules/mlxtend.evaluate/lift_score/","text":"lift_score lift_score(y_target, y_predicted, binary=True, positive_label=1) Lift measures the degree to which the predictions of a classification model are better than randomly-generated predictions. The in terms of True Positives (TP), True Negatives (TN), False Positives (FP), and False Negatives (FN), the lift score is computed as: [ TP / (TP+FP) ] / [ (TP+FN) / (TP+TN+FP+FN) ] Parameters y_target : array-like, shape=[n_samples] True class labels. y_predicted : array-like, shape=[n_samples] Predicted class labels. binary : bool (default: True) Maps a multi-class problem onto a binary, where the positive class is 1 and all other classes are 0. positive_label : int (default: 0) Class label of the positive class. Returns score : float Lift score in the range [0, infinity] Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/lift_score/","title":"Lift score"},{"location":"api_modules/mlxtend.evaluate/lift_score/#lift_score","text":"lift_score(y_target, y_predicted, binary=True, positive_label=1) Lift measures the degree to which the predictions of a classification model are better than randomly-generated predictions. The in terms of True Positives (TP), True Negatives (TN), False Positives (FP), and False Negatives (FN), the lift score is computed as: [ TP / (TP+FP) ] / [ (TP+FN) / (TP+TN+FP+FN) ] Parameters y_target : array-like, shape=[n_samples] True class labels. y_predicted : array-like, shape=[n_samples] Predicted class labels. binary : bool (default: True) Maps a multi-class problem onto a binary, where the positive class is 1 and all other classes are 0. positive_label : int (default: 0) Class label of the positive class. Returns score : float Lift score in the range [0, infinity] Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/lift_score/","title":"lift_score"},{"location":"api_modules/mlxtend.evaluate/mcnemar/","text":"mcnemar mcnemar(ary, corrected=True, exact=False) McNemar test for paired nominal data Parameters ary : array-like, shape=[2, 2] 2 x 2 contigency table (as returned by evaluate.mcnemar_table), where a: ary[0, 0]: # of samples that both models predicted correctly b: ary[0, 1]: # of samples that model 1 got right and model 2 got wrong c: ary[1, 0]: # of samples that model 2 got right and model 1 got wrong d: aryCell [1, 1]: # of samples that both models predicted incorrectly corrected : array-like, shape=[n_samples] (default: True) Uses Edward's continuity correction for chi-squared if True exact : bool, (default: False) If True , uses an exact binomial test comparing b to a binomial distribution with n = b + c and p = 0.5. It is highly recommended to use exact=True for sample sizes < 25 since chi-squared is not well-approximated by the chi-squared distribution! Returns chi2, p : float or None, float Returns the chi-squared value and the p-value; if exact=True (default: False ), chi2 is None Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/mcnemar/","title":"Mcnemar"},{"location":"api_modules/mlxtend.evaluate/mcnemar/#mcnemar","text":"mcnemar(ary, corrected=True, exact=False) McNemar test for paired nominal data Parameters ary : array-like, shape=[2, 2] 2 x 2 contigency table (as returned by evaluate.mcnemar_table), where a: ary[0, 0]: # of samples that both models predicted correctly b: ary[0, 1]: # of samples that model 1 got right and model 2 got wrong c: ary[1, 0]: # of samples that model 2 got right and model 1 got wrong d: aryCell [1, 1]: # of samples that both models predicted incorrectly corrected : array-like, shape=[n_samples] (default: True) Uses Edward's continuity correction for chi-squared if True exact : bool, (default: False) If True , uses an exact binomial test comparing b to a binomial distribution with n = b + c and p = 0.5. It is highly recommended to use exact=True for sample sizes < 25 since chi-squared is not well-approximated by the chi-squared distribution! Returns chi2, p : float or None, float Returns the chi-squared value and the p-value; if exact=True (default: False ), chi2 is None Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/mcnemar/","title":"mcnemar"},{"location":"api_modules/mlxtend.evaluate/mcnemar_table/","text":"mcnemar_table mcnemar_table(y_target, y_model1, y_model2) Compute a 2x2 contigency table for McNemar's test. Parameters y_target : array-like, shape=[n_samples] True class labels as 1D NumPy array. y_model1 : array-like, shape=[n_samples] Predicted class labels from model as 1D NumPy array. y_model2 : array-like, shape=[n_samples] Predicted class labels from model 2 as 1D NumPy array. Returns tb : array-like, shape=[2, 2] 2x2 contingency table with the following contents: a: tb[0, 0]: # of samples that both models predicted correctly b: tb[0, 1]: # of samples that model 1 got right and model 2 got wrong c: tb[1, 0]: # of samples that model 2 got right and model 1 got wrong d: tb[1, 1]: # of samples that both models predicted incorrectly Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/mcnemar_table/","title":"Mcnemar table"},{"location":"api_modules/mlxtend.evaluate/mcnemar_table/#mcnemar_table","text":"mcnemar_table(y_target, y_model1, y_model2) Compute a 2x2 contigency table for McNemar's test. Parameters y_target : array-like, shape=[n_samples] True class labels as 1D NumPy array. y_model1 : array-like, shape=[n_samples] Predicted class labels from model as 1D NumPy array. y_model2 : array-like, shape=[n_samples] Predicted class labels from model 2 as 1D NumPy array. Returns tb : array-like, shape=[2, 2] 2x2 contingency table with the following contents: a: tb[0, 0]: # of samples that both models predicted correctly b: tb[0, 1]: # of samples that model 1 got right and model 2 got wrong c: tb[1, 0]: # of samples that model 2 got right and model 1 got wrong d: tb[1, 1]: # of samples that both models predicted incorrectly Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/mcnemar_table/","title":"mcnemar_table"},{"location":"api_modules/mlxtend.evaluate/mcnemar_tables/","text":"mcnemar_tables mcnemar_tables(y_target, y_model_predictions)* Compute multiple 2x2 contigency tables for McNemar's test or Cochran's Q test. Parameters y_target : array-like, shape=[n_samples] True class labels as 1D NumPy array. y_model_predictions : array-like, shape=[n_samples] Predicted class labels for a model. Returns tables : dict Dictionary of NumPy arrays with shape=[2, 2]. Each dictionary key names the two models to be compared based on the order the models were passed as *y_model_predictions . The number of dictionary entries is equal to the number of pairwise combinations between the m models, i.e., \"m choose 2.\" For example the following target array (containing the true labels) and 3 models y_true = np.array([0, 0, 0, 0, 0, 1, 1, 1, 1, 1]) y_mod0 = np.array([0, 1, 0, 0, 0, 1, 1, 0, 0, 0]) y_mod1 = np.array([0, 0, 1, 1, 0, 1, 1, 0, 0, 0]) y_mod2 = np.array([0, 1, 1, 1, 0, 1, 0, 0, 0, 0]) would result in the following dictionary: {'model_0 vs model_1': array([[ 4., 1.], [ 2., 3.]]), 'model_0 vs model_2': array([[ 3., 0.], [ 3., 4.]]), 'model_1 vs model_2': array([[ 3., 0.], [ 2., 5.]])} Each array is structured in the following way: tb[0, 0]: # of samples that both models predicted correctly tb[0, 1]: # of samples that model a got right and model b got wrong tb[1, 0]: # of samples that model b got right and model a got wrong tb[1, 1]: # of samples that both models predicted incorrectly Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/mcnemar_tables/","title":"Mcnemar tables"},{"location":"api_modules/mlxtend.evaluate/mcnemar_tables/#mcnemar_tables","text":"mcnemar_tables(y_target, y_model_predictions)* Compute multiple 2x2 contigency tables for McNemar's test or Cochran's Q test. Parameters y_target : array-like, shape=[n_samples] True class labels as 1D NumPy array. y_model_predictions : array-like, shape=[n_samples] Predicted class labels for a model. Returns tables : dict Dictionary of NumPy arrays with shape=[2, 2]. Each dictionary key names the two models to be compared based on the order the models were passed as *y_model_predictions . The number of dictionary entries is equal to the number of pairwise combinations between the m models, i.e., \"m choose 2.\" For example the following target array (containing the true labels) and 3 models y_true = np.array([0, 0, 0, 0, 0, 1, 1, 1, 1, 1]) y_mod0 = np.array([0, 1, 0, 0, 0, 1, 1, 0, 0, 0]) y_mod1 = np.array([0, 0, 1, 1, 0, 1, 1, 0, 0, 0]) y_mod2 = np.array([0, 1, 1, 1, 0, 1, 0, 0, 0, 0]) would result in the following dictionary: {'model_0 vs model_1': array([[ 4., 1.], [ 2., 3.]]), 'model_0 vs model_2': array([[ 3., 0.], [ 3., 4.]]), 'model_1 vs model_2': array([[ 3., 0.], [ 2., 5.]])} Each array is structured in the following way: tb[0, 0]: # of samples that both models predicted correctly tb[0, 1]: # of samples that model a got right and model b got wrong tb[1, 0]: # of samples that model b got right and model a got wrong tb[1, 1]: # of samples that both models predicted incorrectly Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/mcnemar_tables/","title":"mcnemar_tables"},{"location":"api_modules/mlxtend.evaluate/paired_ttest_5x2cv/","text":"paired_ttest_5x2cv paired_ttest_5x2cv(estimator1, estimator2, X, y, scoring=None, random_seed=None) Implements the 5x2cv paired t test proposed by Dieterrich (1998) to compare the performance of two models. Parameters estimator1 : scikit-learn classifier or regressor estimator2 : scikit-learn classifier or regressor X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values. scoring : str, callable, or None (default: None) If None (default), uses 'accuracy' for sklearn classifiers and 'r2' for sklearn regressors. If str, uses a sklearn scoring metric string identifier, for example {accuracy, f1, precision, recall, roc_auc} for classifiers, {'mean_absolute_error', 'mean_squared_error'/'neg_mean_squared_error', 'median_absolute_error', 'r2'} for regressors. If a callable object or function is provided, it has to be conform with sklearn's signature scorer(estimator, X, y) ; see https://scikit-learn.org/stable/modules/generated/sklearn.metrics.make_scorer.html for more information. random_seed : int or None (default: None) Random seed for creating the test/train splits. Returns t : float The t-statistic pvalue : float Two-tailed p-value. If the chosen significance level is larger than the p-value, we reject the null hypothesis and accept that there are significant differences in the two compared models. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/paired_ttest_5x2cv/","title":"Paired ttest 5x2cv"},{"location":"api_modules/mlxtend.evaluate/paired_ttest_5x2cv/#paired_ttest_5x2cv","text":"paired_ttest_5x2cv(estimator1, estimator2, X, y, scoring=None, random_seed=None) Implements the 5x2cv paired t test proposed by Dieterrich (1998) to compare the performance of two models. Parameters estimator1 : scikit-learn classifier or regressor estimator2 : scikit-learn classifier or regressor X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values. scoring : str, callable, or None (default: None) If None (default), uses 'accuracy' for sklearn classifiers and 'r2' for sklearn regressors. If str, uses a sklearn scoring metric string identifier, for example {accuracy, f1, precision, recall, roc_auc} for classifiers, {'mean_absolute_error', 'mean_squared_error'/'neg_mean_squared_error', 'median_absolute_error', 'r2'} for regressors. If a callable object or function is provided, it has to be conform with sklearn's signature scorer(estimator, X, y) ; see https://scikit-learn.org/stable/modules/generated/sklearn.metrics.make_scorer.html for more information. random_seed : int or None (default: None) Random seed for creating the test/train splits. Returns t : float The t-statistic pvalue : float Two-tailed p-value. If the chosen significance level is larger than the p-value, we reject the null hypothesis and accept that there are significant differences in the two compared models. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/paired_ttest_5x2cv/","title":"paired_ttest_5x2cv"},{"location":"api_modules/mlxtend.evaluate/paired_ttest_kfold_cv/","text":"paired_ttest_kfold_cv paired_ttest_kfold_cv(estimator1, estimator2, X, y, cv=10, scoring=None, shuffle=False, random_seed=None) Implements the k-fold paired t test procedure to compare the performance of two models. Parameters estimator1 : scikit-learn classifier or regressor estimator2 : scikit-learn classifier or regressor X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values. cv : int (default: 10) Number of splits and iteration for the cross-validation procedure scoring : str, callable, or None (default: None) If None (default), uses 'accuracy' for sklearn classifiers and 'r2' for sklearn regressors. If str, uses a sklearn scoring metric string identifier, for example {accuracy, f1, precision, recall, roc_auc} for classifiers, {'mean_absolute_error', 'mean_squared_error'/'neg_mean_squared_error', 'median_absolute_error', 'r2'} for regressors. If a callable object or function is provided, it has to be conform with sklearn's signature scorer(estimator, X, y) ; see https://scikit-learn.org/stable/modules/generated/sklearn.metrics.make_scorer.html for more information. shuffle : bool (default: True) Whether to shuffle the dataset for generating the k-fold splits. random_seed : int or None (default: None) Random seed for shuffling the dataset for generating the k-fold splits. Ignored if shuffle=False. Returns t : float The t-statistic pvalue : float Two-tailed p-value. If the chosen significance level is larger than the p-value, we reject the null hypothesis and accept that there are significant differences in the two compared models. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/paired_ttest_kfold_cv/","title":"Paired ttest kfold cv"},{"location":"api_modules/mlxtend.evaluate/paired_ttest_kfold_cv/#paired_ttest_kfold_cv","text":"paired_ttest_kfold_cv(estimator1, estimator2, X, y, cv=10, scoring=None, shuffle=False, random_seed=None) Implements the k-fold paired t test procedure to compare the performance of two models. Parameters estimator1 : scikit-learn classifier or regressor estimator2 : scikit-learn classifier or regressor X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values. cv : int (default: 10) Number of splits and iteration for the cross-validation procedure scoring : str, callable, or None (default: None) If None (default), uses 'accuracy' for sklearn classifiers and 'r2' for sklearn regressors. If str, uses a sklearn scoring metric string identifier, for example {accuracy, f1, precision, recall, roc_auc} for classifiers, {'mean_absolute_error', 'mean_squared_error'/'neg_mean_squared_error', 'median_absolute_error', 'r2'} for regressors. If a callable object or function is provided, it has to be conform with sklearn's signature scorer(estimator, X, y) ; see https://scikit-learn.org/stable/modules/generated/sklearn.metrics.make_scorer.html for more information. shuffle : bool (default: True) Whether to shuffle the dataset for generating the k-fold splits. random_seed : int or None (default: None) Random seed for shuffling the dataset for generating the k-fold splits. Ignored if shuffle=False. Returns t : float The t-statistic pvalue : float Two-tailed p-value. If the chosen significance level is larger than the p-value, we reject the null hypothesis and accept that there are significant differences in the two compared models. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/paired_ttest_kfold_cv/","title":"paired_ttest_kfold_cv"},{"location":"api_modules/mlxtend.evaluate/paired_ttest_resampled/","text":"paired_ttest_resampled paired_ttest_resampled(estimator1, estimator2, X, y, num_rounds=30, test_size=0.3, scoring=None, random_seed=None) Implements the resampled paired t test procedure to compare the performance of two models (also called k-hold-out paired t test). Parameters estimator1 : scikit-learn classifier or regressor estimator2 : scikit-learn classifier or regressor X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values. num_rounds : int (default: 30) Number of resampling iterations (i.e., train/test splits) test_size : float or int (default: 0.3) If float, should be between 0.0 and 1.0 and represent the proportion of the dataset to use as a test set. If int, represents the absolute number of test exsamples. scoring : str, callable, or None (default: None) If None (default), uses 'accuracy' for sklearn classifiers and 'r2' for sklearn regressors. If str, uses a sklearn scoring metric string identifier, for example {accuracy, f1, precision, recall, roc_auc} for classifiers, {'mean_absolute_error', 'mean_squared_error'/'neg_mean_squared_error', 'median_absolute_error', 'r2'} for regressors. If a callable object or function is provided, it has to be conform with sklearn's signature scorer(estimator, X, y) ; see https://scikit-learn.org/stable/modules/generated/sklearn.metrics.make_scorer.html for more information. random_seed : int or None (default: None) Random seed for creating the test/train splits. Returns t : float The t-statistic pvalue : float Two-tailed p-value. If the chosen significance level is larger than the p-value, we reject the null hypothesis and accept that there are significant differences in the two compared models. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/paired_ttest_resampled/","title":"Paired ttest resampled"},{"location":"api_modules/mlxtend.evaluate/paired_ttest_resampled/#paired_ttest_resampled","text":"paired_ttest_resampled(estimator1, estimator2, X, y, num_rounds=30, test_size=0.3, scoring=None, random_seed=None) Implements the resampled paired t test procedure to compare the performance of two models (also called k-hold-out paired t test). Parameters estimator1 : scikit-learn classifier or regressor estimator2 : scikit-learn classifier or regressor X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values. num_rounds : int (default: 30) Number of resampling iterations (i.e., train/test splits) test_size : float or int (default: 0.3) If float, should be between 0.0 and 1.0 and represent the proportion of the dataset to use as a test set. If int, represents the absolute number of test exsamples. scoring : str, callable, or None (default: None) If None (default), uses 'accuracy' for sklearn classifiers and 'r2' for sklearn regressors. If str, uses a sklearn scoring metric string identifier, for example {accuracy, f1, precision, recall, roc_auc} for classifiers, {'mean_absolute_error', 'mean_squared_error'/'neg_mean_squared_error', 'median_absolute_error', 'r2'} for regressors. If a callable object or function is provided, it has to be conform with sklearn's signature scorer(estimator, X, y) ; see https://scikit-learn.org/stable/modules/generated/sklearn.metrics.make_scorer.html for more information. random_seed : int or None (default: None) Random seed for creating the test/train splits. Returns t : float The t-statistic pvalue : float Two-tailed p-value. If the chosen significance level is larger than the p-value, we reject the null hypothesis and accept that there are significant differences in the two compared models. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/paired_ttest_resampled/","title":"paired_ttest_resampled"},{"location":"api_modules/mlxtend.evaluate/permutation_test/","text":"permutation_test permutation_test(x, y, func='x_mean != y_mean', method='exact', num_rounds=1000, seed=None, paired=False) Nonparametric permutation test Parameters x : list or numpy array with shape (n_datapoints,) A list or 1D numpy array of the first sample (e.g., the treatment group). y : list or numpy array with shape (n_datapoints,) A list or 1D numpy array of the second sample (e.g., the control group). func : custom function or str (default: 'x_mean != y_mean') function to compute the statistic for the permutation test. - If 'x_mean != y_mean', uses func=lambda x, y: np.abs(np.mean(x) - np.mean(y))) for a two-sided test. - If 'x_mean > y_mean', uses func=lambda x, y: np.mean(x) - np.mean(y)) for a one-sided test. - If 'x_mean < y_mean', uses func=lambda x, y: np.mean(y) - np.mean(x)) for a one-sided test. method : 'approximate' or 'exact' (default: 'exact') If 'exact' (default), all possible permutations are considered. If 'approximate' the number of drawn samples is given by num_rounds . Note that 'exact' is typically not feasible unless the dataset size is relatively small. paired : bool If True, a paired test is performed by only exchanging each datapoint with its associate. num_rounds : int (default: 1000) The number of permutation samples if method='approximate' . seed : int or None (default: None) The random seed for generating permutation samples if method='approximate' . Returns p-value under the null hypothesis Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/permutation_test/","title":"Permutation test"},{"location":"api_modules/mlxtend.evaluate/permutation_test/#permutation_test","text":"permutation_test(x, y, func='x_mean != y_mean', method='exact', num_rounds=1000, seed=None, paired=False) Nonparametric permutation test Parameters x : list or numpy array with shape (n_datapoints,) A list or 1D numpy array of the first sample (e.g., the treatment group). y : list or numpy array with shape (n_datapoints,) A list or 1D numpy array of the second sample (e.g., the control group). func : custom function or str (default: 'x_mean != y_mean') function to compute the statistic for the permutation test. - If 'x_mean != y_mean', uses func=lambda x, y: np.abs(np.mean(x) - np.mean(y))) for a two-sided test. - If 'x_mean > y_mean', uses func=lambda x, y: np.mean(x) - np.mean(y)) for a one-sided test. - If 'x_mean < y_mean', uses func=lambda x, y: np.mean(y) - np.mean(x)) for a one-sided test. method : 'approximate' or 'exact' (default: 'exact') If 'exact' (default), all possible permutations are considered. If 'approximate' the number of drawn samples is given by num_rounds . Note that 'exact' is typically not feasible unless the dataset size is relatively small. paired : bool If True, a paired test is performed by only exchanging each datapoint with its associate. num_rounds : int (default: 1000) The number of permutation samples if method='approximate' . seed : int or None (default: None) The random seed for generating permutation samples if method='approximate' . Returns p-value under the null hypothesis Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/permutation_test/","title":"permutation_test"},{"location":"api_modules/mlxtend.evaluate/proportion_difference/","text":"proportion_difference proportion_difference(proportion_1, proportion_2, n_1, n_2=None) Computes the test statistic and p-value for a difference of proportions test. Parameters proportion_1 : float The first proportion proportion_2 : float The second proportion n_1 : int The sample size of the first test sample n_2 : int or None (default=None) The sample size of the second test sample. If None , n_1 = n_2 . Returns z, p : float or None, float Returns the z-score and the p-value Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/proportion_difference/","title":"Proportion difference"},{"location":"api_modules/mlxtend.evaluate/proportion_difference/#proportion_difference","text":"proportion_difference(proportion_1, proportion_2, n_1, n_2=None) Computes the test statistic and p-value for a difference of proportions test. Parameters proportion_1 : float The first proportion proportion_2 : float The second proportion n_1 : int The sample size of the first test sample n_2 : int or None (default=None) The sample size of the second test sample. If None , n_1 = n_2 . Returns z, p : float or None, float Returns the z-score and the p-value Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/proportion_difference/","title":"proportion_difference"},{"location":"api_modules/mlxtend.evaluate/scoring/","text":"scoring scoring(y_target, y_predicted, metric='error', positive_label=1, unique_labels='auto') Compute a scoring metric for supervised learning. Parameters y_target : array-like, shape=[n_values] True class labels or target values. y_predicted : array-like, shape=[n_values] Predicted class labels or target values. metric : str (default: 'error') Performance metric: 'accuracy': (TP + TN)/(FP + FN + TP + TN) = 1-ERR 'average per-class accuracy': Average per-class accuracy 'average per-class error': Average per-class error 'balanced per-class accuracy': Average per-class accuracy 'balanced per-class error': Average per-class error 'error': (TP + TN)/(FP+ FN + TP + TN) = 1-ACC 'false_positive_rate': FP/N = FP/(FP + TN) 'true_positive_rate': TP/P = TP/(FN + TP) 'true_negative_rate': TN/N = TN/(FP + TN) 'precision': TP/(TP + FP) 'recall': equal to 'true_positive_rate' 'sensitivity': equal to 'true_positive_rate' or 'recall' 'specificity': equal to 'true_negative_rate' 'f1': 2 * (PRE * REC)/(PRE + REC) 'matthews_corr_coef': (TP TN - FP FN) / (sqrt{(TP + FP)( TP + FN )( TN + FP )( TN + FN )}) Where: [TP: True positives, TN = True negatives, TN: True negatives, FN = False negatives] positive_label : int (default: 1) Label of the positive class for binary classification metrics. unique_labels : str or array-like (default: 'auto') If 'auto', deduces the unique class labels from y_target Returns score : float Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/scoring/","title":"Scoring"},{"location":"api_modules/mlxtend.evaluate/scoring/#scoring","text":"scoring(y_target, y_predicted, metric='error', positive_label=1, unique_labels='auto') Compute a scoring metric for supervised learning. Parameters y_target : array-like, shape=[n_values] True class labels or target values. y_predicted : array-like, shape=[n_values] Predicted class labels or target values. metric : str (default: 'error') Performance metric: 'accuracy': (TP + TN)/(FP + FN + TP + TN) = 1-ERR 'average per-class accuracy': Average per-class accuracy 'average per-class error': Average per-class error 'balanced per-class accuracy': Average per-class accuracy 'balanced per-class error': Average per-class error 'error': (TP + TN)/(FP+ FN + TP + TN) = 1-ACC 'false_positive_rate': FP/N = FP/(FP + TN) 'true_positive_rate': TP/P = TP/(FN + TP) 'true_negative_rate': TN/N = TN/(FP + TN) 'precision': TP/(TP + FP) 'recall': equal to 'true_positive_rate' 'sensitivity': equal to 'true_positive_rate' or 'recall' 'specificity': equal to 'true_negative_rate' 'f1': 2 * (PRE * REC)/(PRE + REC) 'matthews_corr_coef': (TP TN - FP FN) / (sqrt{(TP + FP)( TP + FN )( TN + FP )( TN + FN )}) Where: [TP: True positives, TN = True negatives, TN: True negatives, FN = False negatives] positive_label : int (default: 1) Label of the positive class for binary classification metrics. unique_labels : str or array-like (default: 'auto') If 'auto', deduces the unique class labels from y_target Returns score : float Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/scoring/","title":"scoring"},{"location":"api_modules/mlxtend.feature_extraction/LinearDiscriminantAnalysis/","text":"LinearDiscriminantAnalysis LinearDiscriminantAnalysis(n_discriminants=None) Linear Discriminant Analysis Class Parameters n_discriminants : int (default: None) The number of discrimants for transformation. Keeps the original dimensions of the dataset if None . Attributes w_ : array-like, shape=[n_features, n_discriminants] Projection matrix e_vals_ : array-like, shape=[n_features] Eigenvalues in sorted order. e_vecs_ : array-like, shape=[n_features] Eigenvectors in sorted order. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/feature_extraction/LinearDiscriminantAnalysis/ Methods fit(X, y, n_classes=None) Fit the LDA model with X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values. n_classes : int (default: None) A positive integer to declare the number of class labels if not all class labels are present in a partial training set. Gets the number of class labels automatically if None. Returns self : object get_params(deep=True) Get parameters for this estimator. Parameters deep : boolean, optional If True, will return the parameters for this estimator and contained subobjects that are estimators. Returns params : mapping of string to any Parameter names mapped to their values.' adapted from https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/base.py Author: Gael Varoquaux gael.varoquaux@normalesup.org License: BSD 3 clause set_params( params) Set the parameters of this estimator. The method works on simple estimators as well as on nested objects (such as pipelines). The latter have parameters of the form <component>__<parameter> so that it's possible to update each component of a nested object. Returns self adapted from https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/base.py Author: Gael Varoquaux <gael.varoquaux@normalesup.org> License: BSD 3 clause transform(X) Apply the linear transformation on X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns X_projected : np.ndarray, shape = [n_samples, n_discriminants] Projected training vectors.","title":"LinearDiscriminantAnalysis"},{"location":"api_modules/mlxtend.feature_extraction/LinearDiscriminantAnalysis/#lineardiscriminantanalysis","text":"LinearDiscriminantAnalysis(n_discriminants=None) Linear Discriminant Analysis Class Parameters n_discriminants : int (default: None) The number of discrimants for transformation. Keeps the original dimensions of the dataset if None . Attributes w_ : array-like, shape=[n_features, n_discriminants] Projection matrix e_vals_ : array-like, shape=[n_features] Eigenvalues in sorted order. e_vecs_ : array-like, shape=[n_features] Eigenvectors in sorted order. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/feature_extraction/LinearDiscriminantAnalysis/","title":"LinearDiscriminantAnalysis"},{"location":"api_modules/mlxtend.feature_extraction/LinearDiscriminantAnalysis/#methods","text":"fit(X, y, n_classes=None) Fit the LDA model with X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values. n_classes : int (default: None) A positive integer to declare the number of class labels if not all class labels are present in a partial training set. Gets the number of class labels automatically if None. Returns self : object get_params(deep=True) Get parameters for this estimator. Parameters deep : boolean, optional If True, will return the parameters for this estimator and contained subobjects that are estimators. Returns params : mapping of string to any Parameter names mapped to their values.' adapted from https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/base.py Author: Gael Varoquaux gael.varoquaux@normalesup.org License: BSD 3 clause set_params( params) Set the parameters of this estimator. The method works on simple estimators as well as on nested objects (such as pipelines). The latter have parameters of the form <component>__<parameter> so that it's possible to update each component of a nested object. Returns self adapted from https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/base.py Author: Gael Varoquaux <gael.varoquaux@normalesup.org> License: BSD 3 clause transform(X) Apply the linear transformation on X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns X_projected : np.ndarray, shape = [n_samples, n_discriminants] Projected training vectors.","title":"Methods"},{"location":"api_modules/mlxtend.feature_extraction/PrincipalComponentAnalysis/","text":"PrincipalComponentAnalysis PrincipalComponentAnalysis(n_components=None, solver='svd', whitening=False) Principal Component Analysis Class Parameters n_components : int (default: None) The number of principal components for transformation. Keeps the original dimensions of the dataset if None . solver : str (default: 'svd') Method for performing the matrix decomposition. {'eigen', 'svd'} whitening : bool (default: False) Performs whitening such that the covariance matrix of the transformed data will be the identity matrix. Attributes w_ : array-like, shape=[n_features, n_components] Projection matrix e_vals_ : array-like, shape=[n_features] Eigenvalues in sorted order. e_vecs_ : array-like, shape=[n_features] Eigenvectors in sorted order. e_vals_normalized_ : array-like, shape=[n_features] Normalized eigen values such that they sum up to 1. This is equal to what's often referred to as \"explained variance ratios.\" loadings_ : array_like, shape=[n_features, n_features] The factor loadings of the original variables onto the principal components. The columns are the principal components, and the rows are the features loadings. For instance, the first column contains the loadings onto the first principal component. Note that the signs may be flipped depending on whether you use the 'eigen' or 'svd' solver; this does not affect the interpretation of the loadings though. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/feature_extraction/PrincipalComponentAnalysis/ Methods fit(X, y=None) Learn model from training data. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns self : object get_params(deep=True) Get parameters for this estimator. Parameters deep : boolean, optional If True, will return the parameters for this estimator and contained subobjects that are estimators. Returns params : mapping of string to any Parameter names mapped to their values.' adapted from https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/base.py Author: Gael Varoquaux gael.varoquaux@normalesup.org License: BSD 3 clause set_params( params) Set the parameters of this estimator. The method works on simple estimators as well as on nested objects (such as pipelines). The latter have parameters of the form <component>__<parameter> so that it's possible to update each component of a nested object. Returns self adapted from https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/base.py Author: Gael Varoquaux <gael.varoquaux@normalesup.org> License: BSD 3 clause transform(X) Apply the linear transformation on X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns X_projected : np.ndarray, shape = [n_samples, n_components] Projected training vectors.","title":"PrincipalComponentAnalysis"},{"location":"api_modules/mlxtend.feature_extraction/PrincipalComponentAnalysis/#principalcomponentanalysis","text":"PrincipalComponentAnalysis(n_components=None, solver='svd', whitening=False) Principal Component Analysis Class Parameters n_components : int (default: None) The number of principal components for transformation. Keeps the original dimensions of the dataset if None . solver : str (default: 'svd') Method for performing the matrix decomposition. {'eigen', 'svd'} whitening : bool (default: False) Performs whitening such that the covariance matrix of the transformed data will be the identity matrix. Attributes w_ : array-like, shape=[n_features, n_components] Projection matrix e_vals_ : array-like, shape=[n_features] Eigenvalues in sorted order. e_vecs_ : array-like, shape=[n_features] Eigenvectors in sorted order. e_vals_normalized_ : array-like, shape=[n_features] Normalized eigen values such that they sum up to 1. This is equal to what's often referred to as \"explained variance ratios.\" loadings_ : array_like, shape=[n_features, n_features] The factor loadings of the original variables onto the principal components. The columns are the principal components, and the rows are the features loadings. For instance, the first column contains the loadings onto the first principal component. Note that the signs may be flipped depending on whether you use the 'eigen' or 'svd' solver; this does not affect the interpretation of the loadings though. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/feature_extraction/PrincipalComponentAnalysis/","title":"PrincipalComponentAnalysis"},{"location":"api_modules/mlxtend.feature_extraction/PrincipalComponentAnalysis/#methods","text":"fit(X, y=None) Learn model from training data. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns self : object get_params(deep=True) Get parameters for this estimator. Parameters deep : boolean, optional If True, will return the parameters for this estimator and contained subobjects that are estimators. Returns params : mapping of string to any Parameter names mapped to their values.' adapted from https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/base.py Author: Gael Varoquaux gael.varoquaux@normalesup.org License: BSD 3 clause set_params( params) Set the parameters of this estimator. The method works on simple estimators as well as on nested objects (such as pipelines). The latter have parameters of the form <component>__<parameter> so that it's possible to update each component of a nested object. Returns self adapted from https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/base.py Author: Gael Varoquaux <gael.varoquaux@normalesup.org> License: BSD 3 clause transform(X) Apply the linear transformation on X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns X_projected : np.ndarray, shape = [n_samples, n_components] Projected training vectors.","title":"Methods"},{"location":"api_modules/mlxtend.feature_extraction/RBFKernelPCA/","text":"RBFKernelPCA RBFKernelPCA(gamma=15.0, n_components=None, copy_X=True) RBF Kernel Principal Component Analysis for dimensionality reduction. Parameters gamma : float (default: 15.0) Free parameter (coefficient) of the RBF kernel. n_components : int (default: None) The number of principal components for transformation. Keeps the original dimensions of the dataset if None . copy_X : bool (default: True) Copies training data, which is required to compute the projection of new data via the transform method. Uses a reference to X if False. Attributes e_vals_ : array-like, shape=[n_features] Eigenvalues in sorted order. e_vecs_ : array-like, shape=[n_features] Eigenvectors in sorted order. X_projected_ : array-like, shape=[n_samples, n_components] Training samples projected along the component axes. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/feature_extraction/RBFKernelPCA/ Methods fit(X) Learn model from training data. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns self : object get_params(deep=True) Get parameters for this estimator. Parameters deep : boolean, optional If True, will return the parameters for this estimator and contained subobjects that are estimators. Returns params : mapping of string to any Parameter names mapped to their values.' adapted from https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/base.py Author: Gael Varoquaux gael.varoquaux@normalesup.org License: BSD 3 clause set_params( params) Set the parameters of this estimator. The method works on simple estimators as well as on nested objects (such as pipelines). The latter have parameters of the form <component>__<parameter> so that it's possible to update each component of a nested object. Returns self adapted from https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/base.py Author: Gael Varoquaux <gael.varoquaux@normalesup.org> License: BSD 3 clause transform(X) Apply the non-linear transformation on X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns X_projected : np.ndarray, shape = [n_samples, n_components] Projected training vectors.","title":"RBFKernelPCA"},{"location":"api_modules/mlxtend.feature_extraction/RBFKernelPCA/#rbfkernelpca","text":"RBFKernelPCA(gamma=15.0, n_components=None, copy_X=True) RBF Kernel Principal Component Analysis for dimensionality reduction. Parameters gamma : float (default: 15.0) Free parameter (coefficient) of the RBF kernel. n_components : int (default: None) The number of principal components for transformation. Keeps the original dimensions of the dataset if None . copy_X : bool (default: True) Copies training data, which is required to compute the projection of new data via the transform method. Uses a reference to X if False. Attributes e_vals_ : array-like, shape=[n_features] Eigenvalues in sorted order. e_vecs_ : array-like, shape=[n_features] Eigenvectors in sorted order. X_projected_ : array-like, shape=[n_samples, n_components] Training samples projected along the component axes. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/feature_extraction/RBFKernelPCA/","title":"RBFKernelPCA"},{"location":"api_modules/mlxtend.feature_extraction/RBFKernelPCA/#methods","text":"fit(X) Learn model from training data. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns self : object get_params(deep=True) Get parameters for this estimator. Parameters deep : boolean, optional If True, will return the parameters for this estimator and contained subobjects that are estimators. Returns params : mapping of string to any Parameter names mapped to their values.' adapted from https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/base.py Author: Gael Varoquaux gael.varoquaux@normalesup.org License: BSD 3 clause set_params( params) Set the parameters of this estimator. The method works on simple estimators as well as on nested objects (such as pipelines). The latter have parameters of the form <component>__<parameter> so that it's possible to update each component of a nested object. Returns self adapted from https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/base.py Author: Gael Varoquaux <gael.varoquaux@normalesup.org> License: BSD 3 clause transform(X) Apply the non-linear transformation on X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns X_projected : np.ndarray, shape = [n_samples, n_components] Projected training vectors.","title":"Methods"},{"location":"api_modules/mlxtend.feature_selection/ColumnSelector/","text":"ColumnSelector ColumnSelector(cols=None, drop_axis=False) Object for selecting specific columns from a data set. Parameters cols : array-like (default: None) A list specifying the feature indices to be selected. For example, [1, 4, 5] to select the 2nd, 5th, and 6th feature columns, and ['A','C','D'] to select the name of feature columns A, C and D. If None, returns all columns in the array. drop_axis : bool (default=False) Drops last axis if True and the only one column is selected. This is useful, e.g., when the ColumnSelector is used for selecting only one column and the resulting array should be fed to e.g., a scikit-learn column selector. E.g., instead of returning an array with shape (n_samples, 1), drop_axis=True will return an aray with shape (n_samples,). Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/feature_selection/ColumnSelector/ Methods fit(X, y=None) Mock method. Does nothing. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] (default: None) Returns self fit_transform(X, y=None) Return a slice of the input array. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] (default: None) Returns X_slice : shape = [n_samples, k_features] Subset of the feature space where k_features <= n_features get_params(deep=True) Get parameters for this estimator. Parameters deep : bool, default=True If True, will return the parameters for this estimator and contained subobjects that are estimators. Returns params : dict Parameter names mapped to their values. set_params( params) Set the parameters of this estimator. The method works on simple estimators as well as on nested objects (such as :class:`~sklearn.pipeline.Pipeline`). The latter have parameters of the form ``<component>__<parameter>`` so that it's possible to update each component of a nested object. Parameters **params : dict Estimator parameters. Returns self : estimator instance Estimator instance. transform(X, y=None) Return a slice of the input array. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] (default: None) Returns X_slice : shape = [n_samples, k_features] Subset of the feature space where k_features <= n_features","title":"ColumnSelector"},{"location":"api_modules/mlxtend.feature_selection/ColumnSelector/#columnselector","text":"ColumnSelector(cols=None, drop_axis=False) Object for selecting specific columns from a data set. Parameters cols : array-like (default: None) A list specifying the feature indices to be selected. For example, [1, 4, 5] to select the 2nd, 5th, and 6th feature columns, and ['A','C','D'] to select the name of feature columns A, C and D. If None, returns all columns in the array. drop_axis : bool (default=False) Drops last axis if True and the only one column is selected. This is useful, e.g., when the ColumnSelector is used for selecting only one column and the resulting array should be fed to e.g., a scikit-learn column selector. E.g., instead of returning an array with shape (n_samples, 1), drop_axis=True will return an aray with shape (n_samples,). Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/feature_selection/ColumnSelector/","title":"ColumnSelector"},{"location":"api_modules/mlxtend.feature_selection/ColumnSelector/#methods","text":"fit(X, y=None) Mock method. Does nothing. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] (default: None) Returns self fit_transform(X, y=None) Return a slice of the input array. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] (default: None) Returns X_slice : shape = [n_samples, k_features] Subset of the feature space where k_features <= n_features get_params(deep=True) Get parameters for this estimator. Parameters deep : bool, default=True If True, will return the parameters for this estimator and contained subobjects that are estimators. Returns params : dict Parameter names mapped to their values. set_params( params) Set the parameters of this estimator. The method works on simple estimators as well as on nested objects (such as :class:`~sklearn.pipeline.Pipeline`). The latter have parameters of the form ``<component>__<parameter>`` so that it's possible to update each component of a nested object. Parameters **params : dict Estimator parameters. Returns self : estimator instance Estimator instance. transform(X, y=None) Return a slice of the input array. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] (default: None) Returns X_slice : shape = [n_samples, k_features] Subset of the feature space where k_features <= n_features","title":"Methods"},{"location":"api_modules/mlxtend.feature_selection/ExhaustiveFeatureSelector/","text":"ExhaustiveFeatureSelector ExhaustiveFeatureSelector(estimator, min_features=1, max_features=1, print_progress=True, scoring='accuracy', cv=5, n_jobs=1, pre_dispatch='2 n_jobs', clone_estimator=True, fixed_features=None, feature_groups=None)* Exhaustive Feature Selection for Classification and Regression. (new in v0.4.3) Parameters estimator : scikit-learn classifier or regressor min_features : int (default: 1) Minumum number of features to select max_features : int (default: 1) Maximum number of features to select. If parameter feature_groups is not None, the number of features is equal to the number of feature groups, i.e. len(feature_groups) . For example, if feature_groups = [[0], [1], [2, 3], [4]] , then the max_features value cannot exceed 4. print_progress : bool (default: True) Prints progress as the number of epochs to stderr. scoring : str, (default='accuracy') Scoring metric in {accuracy, f1, precision, recall, roc_auc} for classifiers, {'mean_absolute_error', 'mean_squared_error', 'median_absolute_error', 'r2'} for regressors, or a callable object or function with signature scorer(estimator, X, y) . cv : int (default: 5) Scikit-learn cross-validation generator or int . If estimator is a classifier (or y consists of integer class labels), stratified k-fold is performed, and regular k-fold cross-validation otherwise. No cross-validation if cv is None, False, or 0. n_jobs : int (default: 1) The number of CPUs to use for evaluating different feature subsets in parallel. -1 means 'all CPUs'. pre_dispatch : int, or string (default: '2*n_jobs') Controls the number of jobs that get dispatched during parallel execution if n_jobs > 1 or n_jobs=-1 . Reducing this number can be useful to avoid an explosion of memory consumption when more jobs get dispatched than CPUs can process. This parameter can be: None, in which case all the jobs are immediately created and spawned. Use this for lightweight and fast-running jobs, to avoid delays due to on-demand spawning of the jobs An int, giving the exact number of total jobs that are spawned A string, giving an expression as a function of n_jobs, as in 2*n_jobs clone_estimator : bool (default: True) Clones estimator if True; works with the original estimator instance if False. Set to False if the estimator doesn't implement scikit-learn's set_params and get_params methods. In addition, it is required to set cv=0, and n_jobs=1. fixed_features : tuple (default: None) If not None , the feature indices provided as a tuple will be regarded as fixed by the feature selector. For example, if fixed_features=(1, 3, 7) , the 2nd, 4th, and 8th feature are guaranteed to be present in the solution. Note that if fixed_features is not None , make sure that the number of features to be selected is greater than len(fixed_features) . In other words, ensure that k_features > len(fixed_features) . feature_groups : list or None (default: None) Optional argument for treating certain features as a group. This means, the features within a group are always selected together, never split. For example, feature_groups=[[1], [2], [3, 4, 5]] specifies 3 feature groups.In this case, possible feature selection results with k_features=2 are [[1], [2] , [[1], [3, 4, 5]] , or [[2], [3, 4, 5]] . Feature groups can be useful for interpretability, for example, if features 3, 4, 5 are one-hot encoded features. (For more details, please read the notes at the bottom of this docstring). New in mlxtend v. 0.21.0. Attributes best_idx_ : array-like, shape = [n_predictions] Feature Indices of the selected feature subsets. best_feature_names_ : array-like, shape = [n_predictions] Feature names of the selected feature subsets. If pandas DataFrames are used in the fit method, the feature names correspond to the column names. Otherwise, the feature names are string representation of the feature array indices. New in v 0.13.0. best_score_ : float Cross validation average score of the selected subset. subsets_ : dict A dictionary of selected feature subsets during the exhaustive selection, where the dictionary keys are the lengths k of these feature subsets. The dictionary values are dictionaries themselves with the following keys: 'feature_idx' (tuple of indices of the feature subset) 'feature_names' (tuple of feature names of the feat. subset) 'cv_scores' (list individual cross-validation scores) 'avg_score' (average cross-validation score) Note that if pandas DataFrames are used in the fit method, the 'feature_names' correspond to the column names. Otherwise, the feature names are string representation of the feature array indices. The 'feature_names' is new in v. 0.13.0. Notes (1) If parameter feature_groups is not None, the number of features is equal to the number of feature groups, i.e. len(feature_groups) . For example, if feature_groups = [[0], [1], [2, 3], [4]] , then the max_features value cannot exceed 4. (2) Although two or more individual features may be considered as one group throughout the feature-selection process, it does not mean the individual features of that group have the same impact on the outcome. For instance, in linear regression, the coefficient of the feature 2 and 3 can be different even if they are considered as one group in feature_groups. (3) If both fixed_features and feature_groups are specified, ensure that each feature group contains the fixed_features selection. E.g., for a 3-feature set fixed_features=[0, 1] and feature_groups=[[0, 1], [2]] is valid; fixed_features=[0, 1] and feature_groups=[[0], [1, 2]] is not valid. (4) In case of KeyboardInterrupt, the dictionary subsets may not be completed. If user is still interested in getting the best score, they can use method `finalize_fit`. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/feature_selection/ExhaustiveFeatureSelector/ Methods finalize_fit() None fit(X, y, groups=None, fit_params) Perform feature selection and learn model from training data. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. New in v 0.13.0: pandas DataFrames are now also accepted as argument for X. y : array-like, shape = [n_samples] Target values. groups : array-like, with shape (n_samples,), optional Group labels for the samples used while splitting the dataset into train/test set. Passed to the fit method of the cross-validator. fit_params : dict of string -> object, optional Parameters to pass to to the fit method of classifier. Returns self : object fit_transform(X, y, groups=None, fit_params) Fit to training data and return the best selected features from X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. New in v 0.13.0: pandas DataFrames are now also accepted as argument for X. y : array-like, shape = [n_samples] Target values. groups : array-like, with shape (n_samples,), optional Group labels for the samples used while splitting the dataset into train/test set. Passed to the fit method of the cross-validator. fit_params : dict of string -> object, optional Parameters to pass to to the fit method of classifier. Returns Feature subset of X, shape={n_samples, k_features} get_metric_dict(confidence_interval=0.95) Return metric dictionary Parameters confidence_interval : float (default: 0.95) A positive float between 0.0 and 1.0 to compute the confidence interval bounds of the CV score averages. Returns Dictionary with items where each dictionary value is a list with the number of iterations (number of feature subsets) as its length. The dictionary keys corresponding to these lists are as follows: 'feature_idx': tuple of the indices of the feature subset 'cv_scores': list with individual CV scores 'avg_score': of CV average scores 'std_dev': standard deviation of the CV score average 'std_err': standard error of the CV score average 'ci_bound': confidence interval bound of the CV score average get_params(deep=True) Get parameters for this estimator. Parameters deep : bool, default=True If True, will return the parameters for this estimator and contained subobjects that are estimators. Returns params : dict Parameter names mapped to their values. set_params( params) Set the parameters of this estimator. The method works on simple estimators as well as on nested objects (such as :class:`~sklearn.pipeline.Pipeline`). The latter have parameters of the form ``<component>__<parameter>`` so that it's possible to update each component of a nested object. Parameters **params : dict Estimator parameters. Returns self : estimator instance Estimator instance. transform(X) Return the best selected features from X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. New in v 0.13.0: pandas DataFrames are now also accepted as argument for X. Returns Feature subset of X, shape={n_samples, k_features}","title":"ExhaustiveFeatureSelector"},{"location":"api_modules/mlxtend.feature_selection/ExhaustiveFeatureSelector/#exhaustivefeatureselector","text":"ExhaustiveFeatureSelector(estimator, min_features=1, max_features=1, print_progress=True, scoring='accuracy', cv=5, n_jobs=1, pre_dispatch='2 n_jobs', clone_estimator=True, fixed_features=None, feature_groups=None)* Exhaustive Feature Selection for Classification and Regression. (new in v0.4.3) Parameters estimator : scikit-learn classifier or regressor min_features : int (default: 1) Minumum number of features to select max_features : int (default: 1) Maximum number of features to select. If parameter feature_groups is not None, the number of features is equal to the number of feature groups, i.e. len(feature_groups) . For example, if feature_groups = [[0], [1], [2, 3], [4]] , then the max_features value cannot exceed 4. print_progress : bool (default: True) Prints progress as the number of epochs to stderr. scoring : str, (default='accuracy') Scoring metric in {accuracy, f1, precision, recall, roc_auc} for classifiers, {'mean_absolute_error', 'mean_squared_error', 'median_absolute_error', 'r2'} for regressors, or a callable object or function with signature scorer(estimator, X, y) . cv : int (default: 5) Scikit-learn cross-validation generator or int . If estimator is a classifier (or y consists of integer class labels), stratified k-fold is performed, and regular k-fold cross-validation otherwise. No cross-validation if cv is None, False, or 0. n_jobs : int (default: 1) The number of CPUs to use for evaluating different feature subsets in parallel. -1 means 'all CPUs'. pre_dispatch : int, or string (default: '2*n_jobs') Controls the number of jobs that get dispatched during parallel execution if n_jobs > 1 or n_jobs=-1 . Reducing this number can be useful to avoid an explosion of memory consumption when more jobs get dispatched than CPUs can process. This parameter can be: None, in which case all the jobs are immediately created and spawned. Use this for lightweight and fast-running jobs, to avoid delays due to on-demand spawning of the jobs An int, giving the exact number of total jobs that are spawned A string, giving an expression as a function of n_jobs, as in 2*n_jobs clone_estimator : bool (default: True) Clones estimator if True; works with the original estimator instance if False. Set to False if the estimator doesn't implement scikit-learn's set_params and get_params methods. In addition, it is required to set cv=0, and n_jobs=1. fixed_features : tuple (default: None) If not None , the feature indices provided as a tuple will be regarded as fixed by the feature selector. For example, if fixed_features=(1, 3, 7) , the 2nd, 4th, and 8th feature are guaranteed to be present in the solution. Note that if fixed_features is not None , make sure that the number of features to be selected is greater than len(fixed_features) . In other words, ensure that k_features > len(fixed_features) . feature_groups : list or None (default: None) Optional argument for treating certain features as a group. This means, the features within a group are always selected together, never split. For example, feature_groups=[[1], [2], [3, 4, 5]] specifies 3 feature groups.In this case, possible feature selection results with k_features=2 are [[1], [2] , [[1], [3, 4, 5]] , or [[2], [3, 4, 5]] . Feature groups can be useful for interpretability, for example, if features 3, 4, 5 are one-hot encoded features. (For more details, please read the notes at the bottom of this docstring). New in mlxtend v. 0.21.0. Attributes best_idx_ : array-like, shape = [n_predictions] Feature Indices of the selected feature subsets. best_feature_names_ : array-like, shape = [n_predictions] Feature names of the selected feature subsets. If pandas DataFrames are used in the fit method, the feature names correspond to the column names. Otherwise, the feature names are string representation of the feature array indices. New in v 0.13.0. best_score_ : float Cross validation average score of the selected subset. subsets_ : dict A dictionary of selected feature subsets during the exhaustive selection, where the dictionary keys are the lengths k of these feature subsets. The dictionary values are dictionaries themselves with the following keys: 'feature_idx' (tuple of indices of the feature subset) 'feature_names' (tuple of feature names of the feat. subset) 'cv_scores' (list individual cross-validation scores) 'avg_score' (average cross-validation score) Note that if pandas DataFrames are used in the fit method, the 'feature_names' correspond to the column names. Otherwise, the feature names are string representation of the feature array indices. The 'feature_names' is new in v. 0.13.0. Notes (1) If parameter feature_groups is not None, the number of features is equal to the number of feature groups, i.e. len(feature_groups) . For example, if feature_groups = [[0], [1], [2, 3], [4]] , then the max_features value cannot exceed 4. (2) Although two or more individual features may be considered as one group throughout the feature-selection process, it does not mean the individual features of that group have the same impact on the outcome. For instance, in linear regression, the coefficient of the feature 2 and 3 can be different even if they are considered as one group in feature_groups. (3) If both fixed_features and feature_groups are specified, ensure that each feature group contains the fixed_features selection. E.g., for a 3-feature set fixed_features=[0, 1] and feature_groups=[[0, 1], [2]] is valid; fixed_features=[0, 1] and feature_groups=[[0], [1, 2]] is not valid. (4) In case of KeyboardInterrupt, the dictionary subsets may not be completed. If user is still interested in getting the best score, they can use method `finalize_fit`. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/feature_selection/ExhaustiveFeatureSelector/","title":"ExhaustiveFeatureSelector"},{"location":"api_modules/mlxtend.feature_selection/ExhaustiveFeatureSelector/#methods","text":"finalize_fit() None fit(X, y, groups=None, fit_params) Perform feature selection and learn model from training data. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. New in v 0.13.0: pandas DataFrames are now also accepted as argument for X. y : array-like, shape = [n_samples] Target values. groups : array-like, with shape (n_samples,), optional Group labels for the samples used while splitting the dataset into train/test set. Passed to the fit method of the cross-validator. fit_params : dict of string -> object, optional Parameters to pass to to the fit method of classifier. Returns self : object fit_transform(X, y, groups=None, fit_params) Fit to training data and return the best selected features from X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. New in v 0.13.0: pandas DataFrames are now also accepted as argument for X. y : array-like, shape = [n_samples] Target values. groups : array-like, with shape (n_samples,), optional Group labels for the samples used while splitting the dataset into train/test set. Passed to the fit method of the cross-validator. fit_params : dict of string -> object, optional Parameters to pass to to the fit method of classifier. Returns Feature subset of X, shape={n_samples, k_features} get_metric_dict(confidence_interval=0.95) Return metric dictionary Parameters confidence_interval : float (default: 0.95) A positive float between 0.0 and 1.0 to compute the confidence interval bounds of the CV score averages. Returns Dictionary with items where each dictionary value is a list with the number of iterations (number of feature subsets) as its length. The dictionary keys corresponding to these lists are as follows: 'feature_idx': tuple of the indices of the feature subset 'cv_scores': list with individual CV scores 'avg_score': of CV average scores 'std_dev': standard deviation of the CV score average 'std_err': standard error of the CV score average 'ci_bound': confidence interval bound of the CV score average get_params(deep=True) Get parameters for this estimator. Parameters deep : bool, default=True If True, will return the parameters for this estimator and contained subobjects that are estimators. Returns params : dict Parameter names mapped to their values. set_params( params) Set the parameters of this estimator. The method works on simple estimators as well as on nested objects (such as :class:`~sklearn.pipeline.Pipeline`). The latter have parameters of the form ``<component>__<parameter>`` so that it's possible to update each component of a nested object. Parameters **params : dict Estimator parameters. Returns self : estimator instance Estimator instance. transform(X) Return the best selected features from X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. New in v 0.13.0: pandas DataFrames are now also accepted as argument for X. Returns Feature subset of X, shape={n_samples, k_features}","title":"Methods"},{"location":"api_modules/mlxtend.feature_selection/SequentialFeatureSelector/","text":"SequentialFeatureSelector SequentialFeatureSelector(estimator, k_features=1, forward=True, floating=False, verbose=0, scoring=None, cv=5, n_jobs=1, pre_dispatch='2 n_jobs', clone_estimator=True, fixed_features=None, feature_groups=None)* Sequential Feature Selection for Classification and Regression. Parameters estimator : scikit-learn classifier or regressor k_features : int or tuple or str (default: 1) Number of features to select, where k_features < the full feature set. New in 0.4.2: A tuple containing a min and max value can be provided, and the SFS will consider return any feature combination between min and max that scored highest in cross-validation. For example, the tuple (1, 4) will return any combination from 1 up to 4 features instead of a fixed number of features k. New in 0.8.0: A string argument \"best\" or \"parsimonious\". If \"best\" is provided, the feature selector will return the feature subset with the best cross-validation performance. If \"parsimonious\" is provided as an argument, the smallest feature subset that is within one standard error of the cross-validation performance will be selected. forward : bool (default: True) Forward selection if True, backward selection otherwise floating : bool (default: False) Adds a conditional exclusion/inclusion if True. verbose : int (default: 0), level of verbosity to use in logging. If 0, no output, if 1 number of features in current set, if 2 detailed logging i ncluding timestamp and cv scores at step. scoring : str, callable, or None (default: None) If None (default), uses 'accuracy' for sklearn classifiers and 'r2' for sklearn regressors. If str, uses a sklearn scoring metric string identifier, for example {accuracy, f1, precision, recall, roc_auc} for classifiers, {'mean_absolute_error', 'mean_squared_error'/'neg_mean_squared_error', 'median_absolute_error', 'r2'} for regressors. If a callable object or function is provided, it has to be conform with sklearn's signature scorer(estimator, X, y) ; see https://scikit-learn.org/stable/modules/generated/sklearn.metrics.make_scorer.html for more information. cv : int (default: 5) Integer or iterable yielding train, test splits. If cv is an integer and estimator is a classifier (or y consists of integer class labels) stratified k-fold. Otherwise regular k-fold cross-validation is performed. No cross-validation if cv is None, False, or 0. n_jobs : int (default: 1) The number of CPUs to use for evaluating different feature subsets in parallel. -1 means 'all CPUs'. pre_dispatch : int, or string (default: '2*n_jobs') Controls the number of jobs that get dispatched during parallel execution if n_jobs > 1 or n_jobs=-1 . Reducing this number can be useful to avoid an explosion of memory consumption when more jobs get dispatched than CPUs can process. This parameter can be: None, in which case all the jobs are immediately created and spawned. Use this for lightweight and fast-running jobs, to avoid delays due to on-demand spawning of the jobs An int, giving the exact number of total jobs that are spawned A string, giving an expression as a function of n_jobs, as in 2*n_jobs clone_estimator : bool (default: True) Clones estimator if True; works with the original estimator instance if False. Set to False if the estimator doesn't implement scikit-learn's set_params and get_params methods. In addition, it is required to set cv=0, and n_jobs=1. fixed_features : tuple (default: None) If not None , the feature indices provided as a tuple will be regarded as fixed by the feature selector. For example, if fixed_features=(1, 3, 7) , the 2nd, 4th, and 8th feature are guaranteed to be present in the solution. Note that if fixed_features is not None , make sure that the number of features to be selected is greater than len(fixed_features) . In other words, ensure that k_features > len(fixed_features) . New in mlxtend v. 0.18.0. feature_groups : list or None (default: None) Optional argument for treating certain features as a group. This means, the features within a group are always selected together, never split. For example, feature_groups=[[1], [2], [3, 4, 5]] specifies 3 feature groups. In this case, possible feature selection results with k_features=2 are [[1], [2] , [[1], [3, 4, 5]] , or [[2], [3, 4, 5]] . Feature groups can be useful for interpretability, for example, if features 3, 4, 5 are one-hot encoded features. (For more details, please read the notes at the bottom of this docstring). New in mlxtend v. 0.21.0. Attributes k_feature_idx_ : array-like, shape = [n_predictions] Feature Indices of the selected feature subsets. k_feature_names_ : array-like, shape = [n_predictions] Feature names of the selected feature subsets. If pandas DataFrames are used in the fit method, the feature names correspond to the column names. Otherwise, the feature names are string representation of the feature array indices. New in v 0.13.0. k_score_ : float Cross validation average score of the selected subset. subsets_ : dict A dictionary of selected feature subsets during the sequential selection, where the dictionary keys are the lengths k of these feature subsets. If the parameter feature_groups is not None, the value of key indicates the number of groups that are selected together. The dictionary values are dictionaries themselves with the following keys: 'feature_idx' (tuple of indices of the feature subset) 'feature_names' (tuple of feature names of the feat. subset) 'cv_scores' (list individual cross-validation scores) 'avg_score' (average cross-validation score) Note that if pandas DataFrames are used in the fit method, the 'feature_names' correspond to the column names. Otherwise, the feature names are string representation of the feature array indices. The 'feature_names' is new in v 0.13.0. Notes (1) If parameter feature_groups is not None, the number of features is equal to the number of feature groups, i.e. len(feature_groups) . For example, if feature_groups = [[0], [1], [2, 3], [4]] , then the max_features value cannot exceed 4. (2) Although two or more individual features may be considered as one group throughout the feature-selection process, it does not mean the individual features of that group have the same impact on the outcome. For instance, in linear regression, the coefficient of the feature 2 and 3 can be different even if they are considered as one group in feature_groups. (3) If both fixed_features and feature_groups are specified, ensure that each feature group contains the fixed_features selection. E.g., for a 3-feature set fixed_features=[0, 1] and feature_groups=[[0, 1], [2]] is valid; fixed_features=[0, 1] and feature_groups=[[0], [1, 2]] is not valid. (4) In case of KeyboardInterrupt, the dictionary subsets may not be completed. If user is still interested in getting the best score, they can use method `finalize_fit`. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/feature_selection/SequentialFeatureSelector/ Methods finalize_fit() None fit(X, y, groups=None, fit_params) Perform feature selection and learn model from training data. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. New in v 0.13.0: pandas DataFrames are now also accepted as argument for X. y : array-like, shape = [n_samples] Target values. New in v 0.13.0: pandas DataFrames are now also accepted as argument for y. groups : array-like, with shape (n_samples,), optional Group labels for the samples used while splitting the dataset into train/test set. Passed to the fit method of the cross-validator. fit_params : various, optional Additional parameters that are being passed to the estimator. For example, sample_weights=weights . Returns self : object fit_transform(X, y, groups=None, fit_params) Fit to training data then reduce X to its most important features. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. New in v 0.13.0: pandas DataFrames are now also accepted as argument for X. y : array-like, shape = [n_samples] Target values. New in v 0.13.0: a pandas Series are now also accepted as argument for y. groups : array-like, with shape (n_samples,), optional Group labels for the samples used while splitting the dataset into train/test set. Passed to the fit method of the cross-validator. fit_params : various, optional Additional parameters that are being passed to the estimator. For example, sample_weights=weights . Returns Reduced feature subset of X, shape={n_samples, k_features} generate_error_message_k_features(name) None get_metric_dict(confidence_interval=0.95) Return metric dictionary Parameters confidence_interval : float (default: 0.95) A positive float between 0.0 and 1.0 to compute the confidence interval bounds of the CV score averages. Returns Dictionary with items where each dictionary value is a list with the number of iterations (number of feature subsets) as its length. The dictionary keys corresponding to these lists are as follows: 'feature_idx': tuple of the indices of the feature subset 'cv_scores': list with individual CV scores 'avg_score': of CV average scores 'std_dev': standard deviation of the CV score average 'std_err': standard error of the CV score average 'ci_bound': confidence interval bound of the CV score average get_params(deep=True) Get parameters for this estimator. Parameters deep : bool, default=True If True, will return the parameters for this estimator and contained subobjects that are estimators. Returns params : dict Parameter names mapped to their values. set_params( params) Set the parameters of this estimator. Valid parameter keys can be listed with get_params() . Returns self transform(X) Reduce X to its most important features. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. New in v 0.13.0: pandas DataFrames are now also accepted as argument for X. Returns Reduced feature subset of X, shape={n_samples, k_features} Properties named_estimators Returns List of named estimator tuples, like [('svc', SVC(...))]","title":"SequentialFeatureSelector"},{"location":"api_modules/mlxtend.feature_selection/SequentialFeatureSelector/#sequentialfeatureselector","text":"SequentialFeatureSelector(estimator, k_features=1, forward=True, floating=False, verbose=0, scoring=None, cv=5, n_jobs=1, pre_dispatch='2 n_jobs', clone_estimator=True, fixed_features=None, feature_groups=None)* Sequential Feature Selection for Classification and Regression. Parameters estimator : scikit-learn classifier or regressor k_features : int or tuple or str (default: 1) Number of features to select, where k_features < the full feature set. New in 0.4.2: A tuple containing a min and max value can be provided, and the SFS will consider return any feature combination between min and max that scored highest in cross-validation. For example, the tuple (1, 4) will return any combination from 1 up to 4 features instead of a fixed number of features k. New in 0.8.0: A string argument \"best\" or \"parsimonious\". If \"best\" is provided, the feature selector will return the feature subset with the best cross-validation performance. If \"parsimonious\" is provided as an argument, the smallest feature subset that is within one standard error of the cross-validation performance will be selected. forward : bool (default: True) Forward selection if True, backward selection otherwise floating : bool (default: False) Adds a conditional exclusion/inclusion if True. verbose : int (default: 0), level of verbosity to use in logging. If 0, no output, if 1 number of features in current set, if 2 detailed logging i ncluding timestamp and cv scores at step. scoring : str, callable, or None (default: None) If None (default), uses 'accuracy' for sklearn classifiers and 'r2' for sklearn regressors. If str, uses a sklearn scoring metric string identifier, for example {accuracy, f1, precision, recall, roc_auc} for classifiers, {'mean_absolute_error', 'mean_squared_error'/'neg_mean_squared_error', 'median_absolute_error', 'r2'} for regressors. If a callable object or function is provided, it has to be conform with sklearn's signature scorer(estimator, X, y) ; see https://scikit-learn.org/stable/modules/generated/sklearn.metrics.make_scorer.html for more information. cv : int (default: 5) Integer or iterable yielding train, test splits. If cv is an integer and estimator is a classifier (or y consists of integer class labels) stratified k-fold. Otherwise regular k-fold cross-validation is performed. No cross-validation if cv is None, False, or 0. n_jobs : int (default: 1) The number of CPUs to use for evaluating different feature subsets in parallel. -1 means 'all CPUs'. pre_dispatch : int, or string (default: '2*n_jobs') Controls the number of jobs that get dispatched during parallel execution if n_jobs > 1 or n_jobs=-1 . Reducing this number can be useful to avoid an explosion of memory consumption when more jobs get dispatched than CPUs can process. This parameter can be: None, in which case all the jobs are immediately created and spawned. Use this for lightweight and fast-running jobs, to avoid delays due to on-demand spawning of the jobs An int, giving the exact number of total jobs that are spawned A string, giving an expression as a function of n_jobs, as in 2*n_jobs clone_estimator : bool (default: True) Clones estimator if True; works with the original estimator instance if False. Set to False if the estimator doesn't implement scikit-learn's set_params and get_params methods. In addition, it is required to set cv=0, and n_jobs=1. fixed_features : tuple (default: None) If not None , the feature indices provided as a tuple will be regarded as fixed by the feature selector. For example, if fixed_features=(1, 3, 7) , the 2nd, 4th, and 8th feature are guaranteed to be present in the solution. Note that if fixed_features is not None , make sure that the number of features to be selected is greater than len(fixed_features) . In other words, ensure that k_features > len(fixed_features) . New in mlxtend v. 0.18.0. feature_groups : list or None (default: None) Optional argument for treating certain features as a group. This means, the features within a group are always selected together, never split. For example, feature_groups=[[1], [2], [3, 4, 5]] specifies 3 feature groups. In this case, possible feature selection results with k_features=2 are [[1], [2] , [[1], [3, 4, 5]] , or [[2], [3, 4, 5]] . Feature groups can be useful for interpretability, for example, if features 3, 4, 5 are one-hot encoded features. (For more details, please read the notes at the bottom of this docstring). New in mlxtend v. 0.21.0. Attributes k_feature_idx_ : array-like, shape = [n_predictions] Feature Indices of the selected feature subsets. k_feature_names_ : array-like, shape = [n_predictions] Feature names of the selected feature subsets. If pandas DataFrames are used in the fit method, the feature names correspond to the column names. Otherwise, the feature names are string representation of the feature array indices. New in v 0.13.0. k_score_ : float Cross validation average score of the selected subset. subsets_ : dict A dictionary of selected feature subsets during the sequential selection, where the dictionary keys are the lengths k of these feature subsets. If the parameter feature_groups is not None, the value of key indicates the number of groups that are selected together. The dictionary values are dictionaries themselves with the following keys: 'feature_idx' (tuple of indices of the feature subset) 'feature_names' (tuple of feature names of the feat. subset) 'cv_scores' (list individual cross-validation scores) 'avg_score' (average cross-validation score) Note that if pandas DataFrames are used in the fit method, the 'feature_names' correspond to the column names. Otherwise, the feature names are string representation of the feature array indices. The 'feature_names' is new in v 0.13.0. Notes (1) If parameter feature_groups is not None, the number of features is equal to the number of feature groups, i.e. len(feature_groups) . For example, if feature_groups = [[0], [1], [2, 3], [4]] , then the max_features value cannot exceed 4. (2) Although two or more individual features may be considered as one group throughout the feature-selection process, it does not mean the individual features of that group have the same impact on the outcome. For instance, in linear regression, the coefficient of the feature 2 and 3 can be different even if they are considered as one group in feature_groups. (3) If both fixed_features and feature_groups are specified, ensure that each feature group contains the fixed_features selection. E.g., for a 3-feature set fixed_features=[0, 1] and feature_groups=[[0, 1], [2]] is valid; fixed_features=[0, 1] and feature_groups=[[0], [1, 2]] is not valid. (4) In case of KeyboardInterrupt, the dictionary subsets may not be completed. If user is still interested in getting the best score, they can use method `finalize_fit`. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/feature_selection/SequentialFeatureSelector/","title":"SequentialFeatureSelector"},{"location":"api_modules/mlxtend.feature_selection/SequentialFeatureSelector/#methods","text":"finalize_fit() None fit(X, y, groups=None, fit_params) Perform feature selection and learn model from training data. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. New in v 0.13.0: pandas DataFrames are now also accepted as argument for X. y : array-like, shape = [n_samples] Target values. New in v 0.13.0: pandas DataFrames are now also accepted as argument for y. groups : array-like, with shape (n_samples,), optional Group labels for the samples used while splitting the dataset into train/test set. Passed to the fit method of the cross-validator. fit_params : various, optional Additional parameters that are being passed to the estimator. For example, sample_weights=weights . Returns self : object fit_transform(X, y, groups=None, fit_params) Fit to training data then reduce X to its most important features. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. New in v 0.13.0: pandas DataFrames are now also accepted as argument for X. y : array-like, shape = [n_samples] Target values. New in v 0.13.0: a pandas Series are now also accepted as argument for y. groups : array-like, with shape (n_samples,), optional Group labels for the samples used while splitting the dataset into train/test set. Passed to the fit method of the cross-validator. fit_params : various, optional Additional parameters that are being passed to the estimator. For example, sample_weights=weights . Returns Reduced feature subset of X, shape={n_samples, k_features} generate_error_message_k_features(name) None get_metric_dict(confidence_interval=0.95) Return metric dictionary Parameters confidence_interval : float (default: 0.95) A positive float between 0.0 and 1.0 to compute the confidence interval bounds of the CV score averages. Returns Dictionary with items where each dictionary value is a list with the number of iterations (number of feature subsets) as its length. The dictionary keys corresponding to these lists are as follows: 'feature_idx': tuple of the indices of the feature subset 'cv_scores': list with individual CV scores 'avg_score': of CV average scores 'std_dev': standard deviation of the CV score average 'std_err': standard error of the CV score average 'ci_bound': confidence interval bound of the CV score average get_params(deep=True) Get parameters for this estimator. Parameters deep : bool, default=True If True, will return the parameters for this estimator and contained subobjects that are estimators. Returns params : dict Parameter names mapped to their values. set_params( params) Set the parameters of this estimator. Valid parameter keys can be listed with get_params() . Returns self transform(X) Reduce X to its most important features. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. New in v 0.13.0: pandas DataFrames are now also accepted as argument for X. Returns Reduced feature subset of X, shape={n_samples, k_features}","title":"Methods"},{"location":"api_modules/mlxtend.feature_selection/SequentialFeatureSelector/#properties","text":"named_estimators Returns List of named estimator tuples, like [('svc', SVC(...))]","title":"Properties"},{"location":"api_modules/mlxtend.file_io/find_filegroups/","text":"find_filegroups find_filegroups(paths, substring='', extensions=None, validity_check=True, ignore_invisible=True, rstrip='', ignore_substring=None) Find and collect files from different directories in a python dictionary. Parameters paths : list Paths of the directories to be searched. Dictionary keys are build from the first directory. substring : str (default: '') Substring that all files have to contain to be considered. extensions : list (default: None) None or list of allowed file extensions for each path. If provided, the number of extensions must match the number of paths . validity_check : bool (default: None) If True , checks if all dictionary values have the same number of file paths. Prints a warning and returns an empty dictionary if the validity check failed. ignore_invisible : bool (default: True) If True , ignores invisible files (i.e., files starting with a period). rstrip : str (default: '') If provided, strips characters from right side of the file base names after splitting the extension. Useful to trim different filenames to a common stem. E.g,. \"abc_d.txt\" and \"abc_d_.csv\" would share the stem \"abc_d\" if rstrip is set to \"_\". ignore_substring : str (default: None) Ignores files that contain the specified substring. Returns groups : dict Dictionary of files paths. Keys are the file names found in the first directory listed in paths (without file extension). Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/file_io/find_filegroups/","title":"Find filegroups"},{"location":"api_modules/mlxtend.file_io/find_filegroups/#find_filegroups","text":"find_filegroups(paths, substring='', extensions=None, validity_check=True, ignore_invisible=True, rstrip='', ignore_substring=None) Find and collect files from different directories in a python dictionary. Parameters paths : list Paths of the directories to be searched. Dictionary keys are build from the first directory. substring : str (default: '') Substring that all files have to contain to be considered. extensions : list (default: None) None or list of allowed file extensions for each path. If provided, the number of extensions must match the number of paths . validity_check : bool (default: None) If True , checks if all dictionary values have the same number of file paths. Prints a warning and returns an empty dictionary if the validity check failed. ignore_invisible : bool (default: True) If True , ignores invisible files (i.e., files starting with a period). rstrip : str (default: '') If provided, strips characters from right side of the file base names after splitting the extension. Useful to trim different filenames to a common stem. E.g,. \"abc_d.txt\" and \"abc_d_.csv\" would share the stem \"abc_d\" if rstrip is set to \"_\". ignore_substring : str (default: None) Ignores files that contain the specified substring. Returns groups : dict Dictionary of files paths. Keys are the file names found in the first directory listed in paths (without file extension). Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/file_io/find_filegroups/","title":"find_filegroups"},{"location":"api_modules/mlxtend.file_io/find_files/","text":"find_files find_files(substring, path, recursive=False, check_ext=None, ignore_invisible=True, ignore_substring=None) Find files in a directory based on substring matching. Parameters substring : str Substring of the file to be matched. path : str Path where to look. recursive : bool If true, searches subdirectories recursively. check_ext : str If string (e.g., '.txt'), only returns files that match the specified file extension. ignore_invisible : bool If True , ignores invisible files (i.e., files starting with a period). ignore_substring : str Ignores files that contain the specified substring. Returns results : list List of the matched files. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/file_io/find_files/","title":"Find files"},{"location":"api_modules/mlxtend.file_io/find_files/#find_files","text":"find_files(substring, path, recursive=False, check_ext=None, ignore_invisible=True, ignore_substring=None) Find files in a directory based on substring matching. Parameters substring : str Substring of the file to be matched. path : str Path where to look. recursive : bool If true, searches subdirectories recursively. check_ext : str If string (e.g., '.txt'), only returns files that match the specified file extension. ignore_invisible : bool If True , ignores invisible files (i.e., files starting with a period). ignore_substring : str Ignores files that contain the specified substring. Returns results : list List of the matched files. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/file_io/find_files/","title":"find_files"},{"location":"api_modules/mlxtend.frequent_patterns/apriori/","text":"apriori apriori(df, min_support=0.5, use_colnames=False, max_len=None, verbose=0, low_memory=False) Get frequent itemsets from a one-hot DataFrame Parameters df : pandas DataFrame pandas DataFrame the encoded format. Also supports DataFrames with sparse data; for more info, please see (https://pandas.pydata.org/pandas-docs/stable/ user_guide/sparse.html#sparse-data-structures) Please note that the old pandas SparseDataFrame format is no longer supported in mlxtend >= 0.17.2. The allowed values are either 0/1 or True/False. For example, Apple Bananas Beer Chicken Milk Rice 0 True False True True False True 1 True False True False False True 2 True False True False False False 3 True True False False False False 4 False False True True True True 5 False False True False True True 6 False False True False True False 7 True True False False False False min_support : float (default: 0.5) A float between 0 and 1 for minumum support of the itemsets returned. The support is computed as the fraction transactions_where_item(s)_occur / total_transactions . use_colnames : bool (default: False) If True , uses the DataFrames' column names in the returned DataFrame instead of column indices. max_len : int (default: None) Maximum length of the itemsets generated. If None (default) all possible itemsets lengths (under the apriori condition) are evaluated. verbose : int (default: 0) Shows the number of iterations if >= 1 and low_memory is True . If =1 and low_memory is False , shows the number of combinations. low_memory : bool (default: False) If True , uses an iterator to search for combinations above min_support . Note that while low_memory=True should only be used for large dataset if memory resources are limited, because this implementation is approx. 3-6x slower than the default. Returns pandas DataFrame with columns ['support', 'itemsets'] of all itemsets that are >= min_support and < than max_len (if max_len is not None). Each itemset in the 'itemsets' column is of type frozenset , which is a Python built-in type that behaves similarly to sets except that it is immutable (For more info, see https://docs.python.org/3.6/library/stdtypes.html#frozenset). Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/frequent_patterns/apriori/","title":"Apriori"},{"location":"api_modules/mlxtend.frequent_patterns/apriori/#apriori","text":"apriori(df, min_support=0.5, use_colnames=False, max_len=None, verbose=0, low_memory=False) Get frequent itemsets from a one-hot DataFrame Parameters df : pandas DataFrame pandas DataFrame the encoded format. Also supports DataFrames with sparse data; for more info, please see (https://pandas.pydata.org/pandas-docs/stable/ user_guide/sparse.html#sparse-data-structures) Please note that the old pandas SparseDataFrame format is no longer supported in mlxtend >= 0.17.2. The allowed values are either 0/1 or True/False. For example, Apple Bananas Beer Chicken Milk Rice 0 True False True True False True 1 True False True False False True 2 True False True False False False 3 True True False False False False 4 False False True True True True 5 False False True False True True 6 False False True False True False 7 True True False False False False min_support : float (default: 0.5) A float between 0 and 1 for minumum support of the itemsets returned. The support is computed as the fraction transactions_where_item(s)_occur / total_transactions . use_colnames : bool (default: False) If True , uses the DataFrames' column names in the returned DataFrame instead of column indices. max_len : int (default: None) Maximum length of the itemsets generated. If None (default) all possible itemsets lengths (under the apriori condition) are evaluated. verbose : int (default: 0) Shows the number of iterations if >= 1 and low_memory is True . If =1 and low_memory is False , shows the number of combinations. low_memory : bool (default: False) If True , uses an iterator to search for combinations above min_support . Note that while low_memory=True should only be used for large dataset if memory resources are limited, because this implementation is approx. 3-6x slower than the default. Returns pandas DataFrame with columns ['support', 'itemsets'] of all itemsets that are >= min_support and < than max_len (if max_len is not None). Each itemset in the 'itemsets' column is of type frozenset , which is a Python built-in type that behaves similarly to sets except that it is immutable (For more info, see https://docs.python.org/3.6/library/stdtypes.html#frozenset). Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/frequent_patterns/apriori/","title":"apriori"},{"location":"api_modules/mlxtend.frequent_patterns/association_rules/","text":"association_rules association_rules(df, metric='confidence', min_threshold=0.8, support_only=False) Generates a DataFrame of association rules including the metrics 'score', 'confidence', and 'lift' Parameters df : pandas DataFrame pandas DataFrame of frequent itemsets with columns ['support', 'itemsets'] metric : string (default: 'confidence') Metric to evaluate if a rule is of interest. Automatically set to 'support' if support_only=True . Otherwise, supported metrics are 'support', 'confidence', 'lift', 'leverage', 'conviction' and 'zhangs_metric' These metrics are computed as follows: - support(A->C) = support(A+C) [aka 'support'], range: [0, 1] - confidence(A->C) = support(A+C) / support(A), range: [0, 1] - lift(A->C) = confidence(A->C) / support(C), range: [0, inf] - leverage(A->C) = support(A->C) - support(A)*support(C), range: [-1, 1] - conviction = [1 - support(C)] / [1 - confidence(A->C)], range: [0, inf] - zhangs_metric(A->C) = leverage(A->C) / max(support(A->C)*(1-support(A)), support(A)*(support(C)-support(A->C))) range: [-1,1] min_threshold : float (default: 0.8) Minimal threshold for the evaluation metric, via the metric parameter, to decide whether a candidate rule is of interest. support_only : bool (default: False) Only computes the rule support and fills the other metric columns with NaNs. This is useful if: a) the input DataFrame is incomplete, e.g., does not contain support values for all rule antecedents and consequents b) you simply want to speed up the computation because you don't need the other metrics. Returns pandas DataFrame with columns \"antecedents\" and \"consequents\" that store itemsets, plus the scoring metric columns: \"antecedent support\", \"consequent support\", \"support\", \"confidence\", \"lift\", \"leverage\", \"conviction\" of all rules for which metric(rule) >= min_threshold. Each entry in the \"antecedents\" and \"consequents\" columns are of type frozenset , which is a Python built-in type that behaves similarly to sets except that it is immutable (For more info, see https://docs.python.org/3.6/library/stdtypes.html#frozenset). Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/frequent_patterns/association_rules/","title":"Association rules"},{"location":"api_modules/mlxtend.frequent_patterns/association_rules/#association_rules","text":"association_rules(df, metric='confidence', min_threshold=0.8, support_only=False) Generates a DataFrame of association rules including the metrics 'score', 'confidence', and 'lift' Parameters df : pandas DataFrame pandas DataFrame of frequent itemsets with columns ['support', 'itemsets'] metric : string (default: 'confidence') Metric to evaluate if a rule is of interest. Automatically set to 'support' if support_only=True . Otherwise, supported metrics are 'support', 'confidence', 'lift', 'leverage', 'conviction' and 'zhangs_metric' These metrics are computed as follows: - support(A->C) = support(A+C) [aka 'support'], range: [0, 1] - confidence(A->C) = support(A+C) / support(A), range: [0, 1] - lift(A->C) = confidence(A->C) / support(C), range: [0, inf] - leverage(A->C) = support(A->C) - support(A)*support(C), range: [-1, 1] - conviction = [1 - support(C)] / [1 - confidence(A->C)], range: [0, inf] - zhangs_metric(A->C) = leverage(A->C) / max(support(A->C)*(1-support(A)), support(A)*(support(C)-support(A->C))) range: [-1,1] min_threshold : float (default: 0.8) Minimal threshold for the evaluation metric, via the metric parameter, to decide whether a candidate rule is of interest. support_only : bool (default: False) Only computes the rule support and fills the other metric columns with NaNs. This is useful if: a) the input DataFrame is incomplete, e.g., does not contain support values for all rule antecedents and consequents b) you simply want to speed up the computation because you don't need the other metrics. Returns pandas DataFrame with columns \"antecedents\" and \"consequents\" that store itemsets, plus the scoring metric columns: \"antecedent support\", \"consequent support\", \"support\", \"confidence\", \"lift\", \"leverage\", \"conviction\" of all rules for which metric(rule) >= min_threshold. Each entry in the \"antecedents\" and \"consequents\" columns are of type frozenset , which is a Python built-in type that behaves similarly to sets except that it is immutable (For more info, see https://docs.python.org/3.6/library/stdtypes.html#frozenset). Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/frequent_patterns/association_rules/","title":"association_rules"},{"location":"api_modules/mlxtend.frequent_patterns/fpgrowth/","text":"fpgrowth fpgrowth(df, min_support=0.5, use_colnames=False, max_len=None, verbose=0) Get frequent itemsets from a one-hot DataFrame Parameters df : pandas DataFrame pandas DataFrame the encoded format. Also supports DataFrames with sparse data; for more info, please see https://pandas.pydata.org/pandas-docs/stable/user_guide/sparse.html#sparse-data-structures. Please note that the old pandas SparseDataFrame format is no longer supported in mlxtend >= 0.17.2. The allowed values are either 0/1 or True/False. For example, Apple Bananas Beer Chicken Milk Rice 0 True False True True False True 1 True False True False False True 2 True False True False False False 3 True True False False False False 4 False False True True True True 5 False False True False True True 6 False False True False True False 7 True True False False False False min_support : float (default: 0.5) A float between 0 and 1 for minimum support of the itemsets returned. The support is computed as the fraction transactions_where_item(s)_occur / total_transactions. use_colnames : bool (default: False) If true, uses the DataFrames' column names in the returned DataFrame instead of column indices. max_len : int (default: None) Maximum length of the itemsets generated. If None (default) all possible itemsets lengths are evaluated. verbose : int (default: 0) Shows the stages of conditional tree generation. Returns pandas DataFrame with columns ['support', 'itemsets'] of all itemsets that are >= min_support and < than max_len (if max_len is not None). Each itemset in the 'itemsets' column is of type frozenset , which is a Python built-in type that behaves similarly to sets except that it is immutable (For more info, see https://docs.python.org/3.6/library/stdtypes.html#frozenset). Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/frequent_patterns/fpgrowth/","title":"Fpgrowth"},{"location":"api_modules/mlxtend.frequent_patterns/fpgrowth/#fpgrowth","text":"fpgrowth(df, min_support=0.5, use_colnames=False, max_len=None, verbose=0) Get frequent itemsets from a one-hot DataFrame Parameters df : pandas DataFrame pandas DataFrame the encoded format. Also supports DataFrames with sparse data; for more info, please see https://pandas.pydata.org/pandas-docs/stable/user_guide/sparse.html#sparse-data-structures. Please note that the old pandas SparseDataFrame format is no longer supported in mlxtend >= 0.17.2. The allowed values are either 0/1 or True/False. For example, Apple Bananas Beer Chicken Milk Rice 0 True False True True False True 1 True False True False False True 2 True False True False False False 3 True True False False False False 4 False False True True True True 5 False False True False True True 6 False False True False True False 7 True True False False False False min_support : float (default: 0.5) A float between 0 and 1 for minimum support of the itemsets returned. The support is computed as the fraction transactions_where_item(s)_occur / total_transactions. use_colnames : bool (default: False) If true, uses the DataFrames' column names in the returned DataFrame instead of column indices. max_len : int (default: None) Maximum length of the itemsets generated. If None (default) all possible itemsets lengths are evaluated. verbose : int (default: 0) Shows the stages of conditional tree generation. Returns pandas DataFrame with columns ['support', 'itemsets'] of all itemsets that are >= min_support and < than max_len (if max_len is not None). Each itemset in the 'itemsets' column is of type frozenset , which is a Python built-in type that behaves similarly to sets except that it is immutable (For more info, see https://docs.python.org/3.6/library/stdtypes.html#frozenset). Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/frequent_patterns/fpgrowth/","title":"fpgrowth"},{"location":"api_modules/mlxtend.frequent_patterns/fpmax/","text":"fpmax fpmax(df, min_support=0.5, use_colnames=False, max_len=None, verbose=0) Get maximal frequent itemsets from a one-hot DataFrame Parameters df : pandas DataFrame pandas DataFrame the encoded format. Also supports DataFrames with sparse data; for more info, please see (https://pandas.pydata.org/pandas-docs/stable/ user_guide/sparse.html#sparse-data-structures) Please note that the old pandas SparseDataFrame format is no longer supported in mlxtend >= 0.17.2. The allowed values are either 0/1 or True/False. For example, Apple Bananas Beer Chicken Milk Rice 0 True False True True False True 1 True False True False False True 2 True False True False False False 3 True True False False False False 4 False False True True True True 5 False False True False True True 6 False False True False True False 7 True True False False False False min_support : float (default: 0.5) A float between 0 and 1 for minimum support of the itemsets returned. The support is computed as the fraction transactions_where_item(s)_occur / total_transactions. use_colnames : bool (default: False) If true, uses the DataFrames' column names in the returned DataFrame instead of column indices. max_len : int (default: None) Given the set of all maximal itemsets, return those that are less than max_len . If None (default) all possible itemsets lengths are evaluated. verbose : int (default: 0) Shows the stages of conditional tree generation. Returns pandas DataFrame with columns ['support', 'itemsets'] of all maximal itemsets that are >= min_support and < than max_len (if max_len is not None). Each itemset in the 'itemsets' column is of type frozenset , which is a Python built-in type that behaves similarly to sets except that it is immutable (For more info, see https://docs.python.org/3.6/library/stdtypes.html#frozenset). Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/frequent_patterns/fpmax/","title":"Fpmax"},{"location":"api_modules/mlxtend.frequent_patterns/fpmax/#fpmax","text":"fpmax(df, min_support=0.5, use_colnames=False, max_len=None, verbose=0) Get maximal frequent itemsets from a one-hot DataFrame Parameters df : pandas DataFrame pandas DataFrame the encoded format. Also supports DataFrames with sparse data; for more info, please see (https://pandas.pydata.org/pandas-docs/stable/ user_guide/sparse.html#sparse-data-structures) Please note that the old pandas SparseDataFrame format is no longer supported in mlxtend >= 0.17.2. The allowed values are either 0/1 or True/False. For example, Apple Bananas Beer Chicken Milk Rice 0 True False True True False True 1 True False True False False True 2 True False True False False False 3 True True False False False False 4 False False True True True True 5 False False True False True True 6 False False True False True False 7 True True False False False False min_support : float (default: 0.5) A float between 0 and 1 for minimum support of the itemsets returned. The support is computed as the fraction transactions_where_item(s)_occur / total_transactions. use_colnames : bool (default: False) If true, uses the DataFrames' column names in the returned DataFrame instead of column indices. max_len : int (default: None) Given the set of all maximal itemsets, return those that are less than max_len . If None (default) all possible itemsets lengths are evaluated. verbose : int (default: 0) Shows the stages of conditional tree generation. Returns pandas DataFrame with columns ['support', 'itemsets'] of all maximal itemsets that are >= min_support and < than max_len (if max_len is not None). Each itemset in the 'itemsets' column is of type frozenset , which is a Python built-in type that behaves similarly to sets except that it is immutable (For more info, see https://docs.python.org/3.6/library/stdtypes.html#frozenset). Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/frequent_patterns/fpmax/","title":"fpmax"},{"location":"api_modules/mlxtend.frequent_patterns/hmine/","text":"hmine hmine(df, min_support=0.5, use_colnames=False, max_len=None, verbose=0) -> pandas.core.frame.DataFrame Get frequent itemsets from a one-hot DataFrame Parameters df : pandas DataFrame pandas DataFrame the encoded format. Also supports DataFrames with sparse data; for more info, please see https://pandas.pydata.org/pandas-docs/stable/user_guide/sparse.html#sparse-data-structures. Please note that the old pandas SparseDataFrame format is no longer supported in mlxtend >= 0.17.2. The allowed values are either 0/1 or True/False. For example, Apple Bananas Beer Chicken Milk Rice 0 True False True True False True 1 True False True False False True 2 True False True False False False 3 True True False False False False 4 False False True True True True 5 False False True False True True 6 False False True False True False 7 True True False False False False min_support : float (default: 0.5) A float between 0 and 1 for minimum support of the itemsets returned. The support is computed as the fraction transactions_where_item(s)_occur / total_transactions. use_colnames : bool (default: False) If true, uses the DataFrames' column names in the returned DataFrame instead of column indices. max_len : int (default: None) Maximum length of the itemsets generated. If None (default) all possible itemsets lengths are evaluated. verbose : int (default: 0) Shows the stages of conditional tree generation. Returns pandas DataFrame with columns ['support', 'itemsets'] of all itemsets that are >= min_support and < than max_len (if max_len is not None). Each itemset in the 'itemsets' column is of type frozenset , which is a Python built-in type that behaves similarly to sets except that it is immutable (For more info, see https://docs.python.org/3.6/library/stdtypes.html#frozenset). Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/frequent_patterns/hmine/","title":"Hmine"},{"location":"api_modules/mlxtend.frequent_patterns/hmine/#hmine","text":"hmine(df, min_support=0.5, use_colnames=False, max_len=None, verbose=0) -> pandas.core.frame.DataFrame Get frequent itemsets from a one-hot DataFrame Parameters df : pandas DataFrame pandas DataFrame the encoded format. Also supports DataFrames with sparse data; for more info, please see https://pandas.pydata.org/pandas-docs/stable/user_guide/sparse.html#sparse-data-structures. Please note that the old pandas SparseDataFrame format is no longer supported in mlxtend >= 0.17.2. The allowed values are either 0/1 or True/False. For example, Apple Bananas Beer Chicken Milk Rice 0 True False True True False True 1 True False True False False True 2 True False True False False False 3 True True False False False False 4 False False True True True True 5 False False True False True True 6 False False True False True False 7 True True False False False False min_support : float (default: 0.5) A float between 0 and 1 for minimum support of the itemsets returned. The support is computed as the fraction transactions_where_item(s)_occur / total_transactions. use_colnames : bool (default: False) If true, uses the DataFrames' column names in the returned DataFrame instead of column indices. max_len : int (default: None) Maximum length of the itemsets generated. If None (default) all possible itemsets lengths are evaluated. verbose : int (default: 0) Shows the stages of conditional tree generation. Returns pandas DataFrame with columns ['support', 'itemsets'] of all itemsets that are >= min_support and < than max_len (if max_len is not None). Each itemset in the 'itemsets' column is of type frozenset , which is a Python built-in type that behaves similarly to sets except that it is immutable (For more info, see https://docs.python.org/3.6/library/stdtypes.html#frozenset). Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/frequent_patterns/hmine/","title":"hmine"},{"location":"api_modules/mlxtend.image/EyepadAlign/","text":"EyepadAlign EyepadAlign(verbose=0) Class to align/transform face images to facial landmarks, based on eye alignment. 1. A scaling factor is computed based on distance between the left and right eye, such that the transformed face image will have the same eye distance as a reference face image. 2. A transformation is performed based on the eyes' center point. to align the face based on the reference eye location. 3. Finally, the transformed image is padded with zeros to match the desired final image size. Parameters verbose : int (default=0) Verbose level to display the progress bar and log messages. Setting verbose=1 will print a progress bar upon calling fit_directory . Attributes target_landmarks_ : target landmarks to transform new face images to. Depending on the chosen fit parameters, it can be either (1) assigned to pre-fit shapes, (2) computed from a single face image (3) computed as the mean of face landmarks from all face images in a file directory of face images. eye_distance_ : the distance between left and right eyes in the target landmarks. target_height_ : the height of the transformed output image. target_width_ : the width of the transformed output image. Returns self : object Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/image/eyepad_align/ Methods fit_directory(target_img_dir, target_height, target_width, file_extension='.jpg', pre_check=True) Calculates the average landmarks for all face images in a directory which will then be set as the target landmark set. Arguments target_img_dir : str Directory containing the images target_height : int Expected image height of the images in the directory target_width : int Expected image width of the images in the directory file_extension str (default='.jpg'): File extension of the image files. pre_check Bool (default=True): Checks that each image has the dimensions specificed via target_height and target_width on the whole directory first to identify potential issues that are recommended to be fixed before proceeding. Raises a warning for each image if dimensions differ from the ones specified and expected. Returns self : object fit_image(target_image) Derives facial landmarks from a target image. Arguments target_image : uint8 numpy.array, shape=[width, height, channels] NumPy array representation of the image data. Returns self : object fit_values(target_landmarks, target_width, target_height) Used for determining the eye location from pre-defined landmark arrays, eliminating the need for re-computing the average landmarks on a target image or image directory. Arguments target_landmarks : np.array, shape=(height, width) NumPy array containing the locations of the facial landmarks as determined by mlxtend.image.extract_face_landmarks target_height : int image height target_width : int image width Returns self : object transform(img) transforms a single face image (img) to the target landmarks based on the location of the eyes by scaling, translation and cropping (if needed): (1) Scaling the image so that the distance of the two eyes in the given image (img) matches the distance of the two eyes in the target landmarks. (2) Translation is performed based on the middle point between the two eyes. Arguments img : np.array, shape=(height, width, channels) Input image to be transformed. Returns self : object","title":"EyepadAlign"},{"location":"api_modules/mlxtend.image/EyepadAlign/#eyepadalign","text":"EyepadAlign(verbose=0) Class to align/transform face images to facial landmarks, based on eye alignment. 1. A scaling factor is computed based on distance between the left and right eye, such that the transformed face image will have the same eye distance as a reference face image. 2. A transformation is performed based on the eyes' center point. to align the face based on the reference eye location. 3. Finally, the transformed image is padded with zeros to match the desired final image size. Parameters verbose : int (default=0) Verbose level to display the progress bar and log messages. Setting verbose=1 will print a progress bar upon calling fit_directory . Attributes target_landmarks_ : target landmarks to transform new face images to. Depending on the chosen fit parameters, it can be either (1) assigned to pre-fit shapes, (2) computed from a single face image (3) computed as the mean of face landmarks from all face images in a file directory of face images. eye_distance_ : the distance between left and right eyes in the target landmarks. target_height_ : the height of the transformed output image. target_width_ : the width of the transformed output image. Returns self : object Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/image/eyepad_align/","title":"EyepadAlign"},{"location":"api_modules/mlxtend.image/EyepadAlign/#methods","text":"fit_directory(target_img_dir, target_height, target_width, file_extension='.jpg', pre_check=True) Calculates the average landmarks for all face images in a directory which will then be set as the target landmark set. Arguments target_img_dir : str Directory containing the images target_height : int Expected image height of the images in the directory target_width : int Expected image width of the images in the directory file_extension str (default='.jpg'): File extension of the image files. pre_check Bool (default=True): Checks that each image has the dimensions specificed via target_height and target_width on the whole directory first to identify potential issues that are recommended to be fixed before proceeding. Raises a warning for each image if dimensions differ from the ones specified and expected. Returns self : object fit_image(target_image) Derives facial landmarks from a target image. Arguments target_image : uint8 numpy.array, shape=[width, height, channels] NumPy array representation of the image data. Returns self : object fit_values(target_landmarks, target_width, target_height) Used for determining the eye location from pre-defined landmark arrays, eliminating the need for re-computing the average landmarks on a target image or image directory. Arguments target_landmarks : np.array, shape=(height, width) NumPy array containing the locations of the facial landmarks as determined by mlxtend.image.extract_face_landmarks target_height : int image height target_width : int image width Returns self : object transform(img) transforms a single face image (img) to the target landmarks based on the location of the eyes by scaling, translation and cropping (if needed): (1) Scaling the image so that the distance of the two eyes in the given image (img) matches the distance of the two eyes in the target landmarks. (2) Translation is performed based on the middle point between the two eyes. Arguments img : np.array, shape=(height, width, channels) Input image to be transformed. Returns self : object","title":"Methods"},{"location":"api_modules/mlxtend.image/extract_face_landmarks/","text":"extract_face_landmarks extract_face_landmarks(img, return_dtype= ) Function to extract face landmarks. Note that this function requires an installation of the Python version of the library \"dlib\": https://dlib.net Parameters img : array, shape = [h, w, ?] Numpy array of a face image or imageio.core.util.Array. E.g., img = imageio.core.util.Array(ary) Supported shapes are - 3D tensors with 1 or more color channels, for example, RGB: [h, w, 3] - 2D tensors without color channel, for example, Grayscale: [h, w] return_dtype: the return data-type of the array, default: np.int32. Returns landmarks : numpy.ndarray, shape = [68, 2] A numpy array, where each row contains a landmark/point x-y coordinates. Return None if no face is detected by Dlib. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/image/extract_face_landmarks/","title":"Extract face landmarks"},{"location":"api_modules/mlxtend.image/extract_face_landmarks/#extract_face_landmarks","text":"extract_face_landmarks(img, return_dtype= ) Function to extract face landmarks. Note that this function requires an installation of the Python version of the library \"dlib\": https://dlib.net Parameters img : array, shape = [h, w, ?] Numpy array of a face image or imageio.core.util.Array. E.g., img = imageio.core.util.Array(ary) Supported shapes are - 3D tensors with 1 or more color channels, for example, RGB: [h, w, 3] - 2D tensors without color channel, for example, Grayscale: [h, w] return_dtype: the return data-type of the array, default: np.int32. Returns landmarks : numpy.ndarray, shape = [68, 2] A numpy array, where each row contains a landmark/point x-y coordinates. Return None if no face is detected by Dlib. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/image/extract_face_landmarks/","title":"extract_face_landmarks"},{"location":"api_modules/mlxtend.math/factorial/","text":"factorial factorial(n) None","title":"Factorial"},{"location":"api_modules/mlxtend.math/factorial/#factorial","text":"factorial(n) None","title":"factorial"},{"location":"api_modules/mlxtend.math/num_combinations/","text":"num_combinations num_combinations(n, k, with_replacement=False) Function to calculate the number of possible combinations. Parameters n : int Total number of items. k : int Number of elements of the target itemset. with_replacement : bool (default: False) Allows repeated elements if True. Returns comb : int Number of possible combinations. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/math/num_combinations/","title":"Num combinations"},{"location":"api_modules/mlxtend.math/num_combinations/#num_combinations","text":"num_combinations(n, k, with_replacement=False) Function to calculate the number of possible combinations. Parameters n : int Total number of items. k : int Number of elements of the target itemset. with_replacement : bool (default: False) Allows repeated elements if True. Returns comb : int Number of possible combinations. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/math/num_combinations/","title":"num_combinations"},{"location":"api_modules/mlxtend.math/num_permutations/","text":"num_permutations num_permutations(n, k, with_replacement=False) Function to calculate the number of possible permutations. Parameters n : int Total number of items. k : int Number of elements of the target itemset. with_replacement : bool Allows repeated elements if True. Returns permut : int Number of possible permutations. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/math/num_permutations/","title":"Num permutations"},{"location":"api_modules/mlxtend.math/num_permutations/#num_permutations","text":"num_permutations(n, k, with_replacement=False) Function to calculate the number of possible permutations. Parameters n : int Total number of items. k : int Number of elements of the target itemset. with_replacement : bool Allows repeated elements if True. Returns permut : int Number of possible permutations. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/math/num_permutations/","title":"num_permutations"},{"location":"api_modules/mlxtend.math/vectorspace_dimensionality/","text":"vectorspace_dimensionality vectorspace_dimensionality(ary) Computes the hyper-volume spanned by a vector set Parameters ary : array-like, shape=[num_vectors, num_vectors] An orthogonal set of vectors (arranged as columns in a matrix) Returns dimensions : int An integer indicating the \"dimensionality\" hyper-volume spanned by the vector set Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/math/vectorspace_dimensionality/","title":"Vectorspace dimensionality"},{"location":"api_modules/mlxtend.math/vectorspace_dimensionality/#vectorspace_dimensionality","text":"vectorspace_dimensionality(ary) Computes the hyper-volume spanned by a vector set Parameters ary : array-like, shape=[num_vectors, num_vectors] An orthogonal set of vectors (arranged as columns in a matrix) Returns dimensions : int An integer indicating the \"dimensionality\" hyper-volume spanned by the vector set Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/math/vectorspace_dimensionality/","title":"vectorspace_dimensionality"},{"location":"api_modules/mlxtend.math/vectorspace_orthonormalization/","text":"vectorspace_orthonormalization vectorspace_orthonormalization(ary, eps=1e-13) Transforms a set of column vectors to a orthonormal basis. Given a set of orthogonal vectors, this functions converts such column vectors, arranged in a matrix, into orthonormal basis vectors. Parameters ary : array-like, shape=[num_vectors, num_vectors] An orthogonal set of vectors (arranged as columns in a matrix) eps : float (default: 1e-13) A small tolerance value to determine whether the vector norm is zero or not. Returns arr : array-like, shape=[num_vectors, num_vectors] An orthonormal set of vectors (arranged as columns) Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/math/vectorspace_orthonormalization/","title":"Vectorspace orthonormalization"},{"location":"api_modules/mlxtend.math/vectorspace_orthonormalization/#vectorspace_orthonormalization","text":"vectorspace_orthonormalization(ary, eps=1e-13) Transforms a set of column vectors to a orthonormal basis. Given a set of orthogonal vectors, this functions converts such column vectors, arranged in a matrix, into orthonormal basis vectors. Parameters ary : array-like, shape=[num_vectors, num_vectors] An orthogonal set of vectors (arranged as columns in a matrix) eps : float (default: 1e-13) A small tolerance value to determine whether the vector norm is zero or not. Returns arr : array-like, shape=[num_vectors, num_vectors] An orthonormal set of vectors (arranged as columns) Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/math/vectorspace_orthonormalization/","title":"vectorspace_orthonormalization"},{"location":"api_modules/mlxtend.plotting/category_scatter/","text":"category_scatter category_scatter(x, y, label_col, data, markers='sxo^v', colors=('blue', 'green', 'red', 'purple', 'gray', 'cyan'), alpha=0.7, markersize=20.0, legend_loc='best') Scatter plot to plot categories in different colors/markerstyles. Parameters x : str or int DataFrame column name of the x-axis values or integer for the numpy ndarray column index. y : str DataFrame column name of the y-axis values or integer for the numpy ndarray column index data : Pandas DataFrame object or NumPy ndarray. markers : str Markers that are cycled through the label category. colors : tuple Colors that are cycled through the label category. alpha : float (default: 0.7) Parameter to control the transparency. markersize : float (default` : 20.0) Parameter to control the marker size. legend_loc : str (default: 'best') Location of the plot legend {best, upper left, upper right, lower left, lower right} No legend if legend_loc=False Returns fig : matplotlig.pyplot figure object Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/plotting/category_scatter/","title":"Category scatter"},{"location":"api_modules/mlxtend.plotting/category_scatter/#category_scatter","text":"category_scatter(x, y, label_col, data, markers='sxo^v', colors=('blue', 'green', 'red', 'purple', 'gray', 'cyan'), alpha=0.7, markersize=20.0, legend_loc='best') Scatter plot to plot categories in different colors/markerstyles. Parameters x : str or int DataFrame column name of the x-axis values or integer for the numpy ndarray column index. y : str DataFrame column name of the y-axis values or integer for the numpy ndarray column index data : Pandas DataFrame object or NumPy ndarray. markers : str Markers that are cycled through the label category. colors : tuple Colors that are cycled through the label category. alpha : float (default: 0.7) Parameter to control the transparency. markersize : float (default` : 20.0) Parameter to control the marker size. legend_loc : str (default: 'best') Location of the plot legend {best, upper left, upper right, lower left, lower right} No legend if legend_loc=False Returns fig : matplotlig.pyplot figure object Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/plotting/category_scatter/","title":"category_scatter"},{"location":"api_modules/mlxtend.plotting/checkerboard_plot/","text":"checkerboard_plot checkerboard_plot(ary, cell_colors=('white', 'black'), font_colors=('black', 'white'), fmt='%.1f', figsize=None, row_labels=None, col_labels=None, fontsize=None) Plot a checkerboard table / heatmap via matplotlib. Parameters ary : array-like, shape = [n, m] A 2D Nnumpy array. cell_colors : tuple or list (default: ('white', 'black')) Tuple or list containing the two colors of the checkerboard pattern. font_colors : tuple or list (default: ('black', 'white')) Font colors corresponding to the cell colors. figsize : tuple (default: (2.5, 2.5)) Height and width of the figure fmt : str (default: '%.1f') Python string formatter for cell values. The default '%.1f' results in floats with 1 digit after the decimal point. Use '%d' to show numbers as integers. row_labels : list (default: None) List of the row labels. Uses the array row indices 0 to n by default. col_labels : list (default: None) List of the column labels. Uses the array column indices 0 to m by default. fontsize : int (default: None) Specifies the font size of the checkerboard table. Uses matplotlib's default if None. Returns fig : matplotlib Figure object. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/plotting/checkerboard_plot/","title":"Checkerboard plot"},{"location":"api_modules/mlxtend.plotting/checkerboard_plot/#checkerboard_plot","text":"checkerboard_plot(ary, cell_colors=('white', 'black'), font_colors=('black', 'white'), fmt='%.1f', figsize=None, row_labels=None, col_labels=None, fontsize=None) Plot a checkerboard table / heatmap via matplotlib. Parameters ary : array-like, shape = [n, m] A 2D Nnumpy array. cell_colors : tuple or list (default: ('white', 'black')) Tuple or list containing the two colors of the checkerboard pattern. font_colors : tuple or list (default: ('black', 'white')) Font colors corresponding to the cell colors. figsize : tuple (default: (2.5, 2.5)) Height and width of the figure fmt : str (default: '%.1f') Python string formatter for cell values. The default '%.1f' results in floats with 1 digit after the decimal point. Use '%d' to show numbers as integers. row_labels : list (default: None) List of the row labels. Uses the array row indices 0 to n by default. col_labels : list (default: None) List of the column labels. Uses the array column indices 0 to m by default. fontsize : int (default: None) Specifies the font size of the checkerboard table. Uses matplotlib's default if None. Returns fig : matplotlib Figure object. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/plotting/checkerboard_plot/","title":"checkerboard_plot"},{"location":"api_modules/mlxtend.plotting/ecdf/","text":"ecdf ecdf(x, y_label='ECDF', x_label=None, ax=None, percentile=None, ecdf_color=None, ecdf_marker='o', percentile_color='black', percentile_linestyle='--') Plots an Empirical Cumulative Distribution Function Parameters x : array or list, shape=[n_samples,] Array-like object containing the feature values y_label : str (default='ECDF') Text label for the y-axis x_label : str (default=None) Text label for the x-axis ax : matplotlib.axes.Axes (default: None) An existing matplotlib Axes. Creates one if ax=None percentile : float (default=None) Float between 0 and 1 for plotting a percentile threshold line ecdf_color : matplotlib color (default=None) Color for the ECDF plot; uses matplotlib defaults if None ecdf_marker : matplotlib marker (default='o') Marker style for the ECDF plot percentile_color : matplotlib color (default='black') Color for the percentile threshold if percentile is not None percentile_linestyle : matplotlib linestyle (default='--') Line style for the percentile threshold if percentile is not None Returns ax : matplotlib.axes.Axes object percentile_threshold : float Feature threshold at the percentile or None if percentile=None percentile_count : Number of if percentile is not None Number of samples that have a feature less or equal than the feature threshold at a percentile threshold or None if percentile=None Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/plotting/ecdf/","title":"Ecdf"},{"location":"api_modules/mlxtend.plotting/ecdf/#ecdf","text":"ecdf(x, y_label='ECDF', x_label=None, ax=None, percentile=None, ecdf_color=None, ecdf_marker='o', percentile_color='black', percentile_linestyle='--') Plots an Empirical Cumulative Distribution Function Parameters x : array or list, shape=[n_samples,] Array-like object containing the feature values y_label : str (default='ECDF') Text label for the y-axis x_label : str (default=None) Text label for the x-axis ax : matplotlib.axes.Axes (default: None) An existing matplotlib Axes. Creates one if ax=None percentile : float (default=None) Float between 0 and 1 for plotting a percentile threshold line ecdf_color : matplotlib color (default=None) Color for the ECDF plot; uses matplotlib defaults if None ecdf_marker : matplotlib marker (default='o') Marker style for the ECDF plot percentile_color : matplotlib color (default='black') Color for the percentile threshold if percentile is not None percentile_linestyle : matplotlib linestyle (default='--') Line style for the percentile threshold if percentile is not None Returns ax : matplotlib.axes.Axes object percentile_threshold : float Feature threshold at the percentile or None if percentile=None percentile_count : Number of if percentile is not None Number of samples that have a feature less or equal than the feature threshold at a percentile threshold or None if percentile=None Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/plotting/ecdf/","title":"ecdf"},{"location":"api_modules/mlxtend.plotting/enrichment_plot/","text":"enrichment_plot enrichment_plot(df, colors='bgrkcy', markers=' ', linestyles='-', alpha=0.5, lw=2, where='post', grid=True, count_label='Count', xlim='auto', ylim='auto', invert_axes=False, legend_loc='best', ax=None) Plot stacked barplots Parameters df : pandas.DataFrame A pandas DataFrame where columns represent the different categories. colors: str (default: 'bgrcky') The colors of the bars. markers : str (default: ' ') Matplotlib markerstyles, e.g, 'sov' for square,circle, and triangle markers. linestyles : str (default: '-') Matplotlib linestyles, e.g., '-,--' to cycle normal and dashed lines. Note that the different linestyles need to be separated by commas. alpha : float (default: 0.5) Transparency level from 0.0 to 1.0. lw : int or float (default: 2) Linewidth parameter. where : {'post', 'pre', 'mid'} (default: 'post') Starting location of the steps. grid : bool (default: True ) Plots a grid if True. count_label : str (default: 'Count') Label for the \"Count\"-axis. xlim : 'auto' or array-like [min, max] (default: 'auto') Min and maximum position of the x-axis range. ylim : 'auto' or array-like [min, max] (default: 'auto') Min and maximum position of the y-axis range. invert_axes : bool (default: False) Plots count on the x-axis if True. legend_loc : str (default: 'best') Location of the plot legend {best, upper left, upper right, lower left, lower right} No legend if legend_loc=False ax : matplotlib axis, optional (default: None) Use this axis for plotting or make a new one otherwise Returns ax : matplotlib axis Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/plotting/enrichment_plot/","title":"Enrichment plot"},{"location":"api_modules/mlxtend.plotting/enrichment_plot/#enrichment_plot","text":"enrichment_plot(df, colors='bgrkcy', markers=' ', linestyles='-', alpha=0.5, lw=2, where='post', grid=True, count_label='Count', xlim='auto', ylim='auto', invert_axes=False, legend_loc='best', ax=None) Plot stacked barplots Parameters df : pandas.DataFrame A pandas DataFrame where columns represent the different categories. colors: str (default: 'bgrcky') The colors of the bars. markers : str (default: ' ') Matplotlib markerstyles, e.g, 'sov' for square,circle, and triangle markers. linestyles : str (default: '-') Matplotlib linestyles, e.g., '-,--' to cycle normal and dashed lines. Note that the different linestyles need to be separated by commas. alpha : float (default: 0.5) Transparency level from 0.0 to 1.0. lw : int or float (default: 2) Linewidth parameter. where : {'post', 'pre', 'mid'} (default: 'post') Starting location of the steps. grid : bool (default: True ) Plots a grid if True. count_label : str (default: 'Count') Label for the \"Count\"-axis. xlim : 'auto' or array-like [min, max] (default: 'auto') Min and maximum position of the x-axis range. ylim : 'auto' or array-like [min, max] (default: 'auto') Min and maximum position of the y-axis range. invert_axes : bool (default: False) Plots count on the x-axis if True. legend_loc : str (default: 'best') Location of the plot legend {best, upper left, upper right, lower left, lower right} No legend if legend_loc=False ax : matplotlib axis, optional (default: None) Use this axis for plotting or make a new one otherwise Returns ax : matplotlib axis Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/plotting/enrichment_plot/","title":"enrichment_plot"},{"location":"api_modules/mlxtend.plotting/heatmap/","text":"heatmap heatmap(matrix, hide_spines=False, hide_ticks=False, figsize=None, cmap=None, colorbar=True, row_names=None, column_names=None, column_name_rotation=45, cell_values=True, cell_fmt='.2f', cell_font_size=None, text_color_threshold=None) Plot a heatmap via matplotlib. Parameters conf_mat : array-like, shape = [n_rows, n_columns] And arbitrary 2D array. hide_spines : bool (default: False) Hides axis spines if True. hide_ticks : bool (default: False) Hides axis ticks if True figsize : tuple (default: (2.5, 2.5)) Height and width of the figure cmap : matplotlib colormap (default: None ) Uses matplotlib.pyplot.cm.viridis if None colorbar : bool (default: True) Shows a colorbar if True row_names : array-like, shape = [n_rows] (default: None) List of row names to be used as y-axis tick labels. column_names : array-like, shape = [n_columns] (default: None) List of column names to be used as x-axis tick labels. column_name_rotation : int (default: 45) Number of degrees for rotating column x-tick labels. cell_values : bool (default: True) Plots cell values if True. cell_fmt : string (default: '.2f') Format specification for cell values (if cell_values=True ) cell_font_size : int (default: None) Font size for cell values (if cell_values=True ) text_color_threshold : float (default: None) Threshold for the black/white text threshold of the text annotation. Default (None) tried to infer a good threshold automatically using np.max(normed_matrix) / 2 . Returns fig, ax : matplotlib.pyplot subplot objects Figure and axis elements of the subplot. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/plotting/heatmap/","title":"Heatmap"},{"location":"api_modules/mlxtend.plotting/heatmap/#heatmap","text":"heatmap(matrix, hide_spines=False, hide_ticks=False, figsize=None, cmap=None, colorbar=True, row_names=None, column_names=None, column_name_rotation=45, cell_values=True, cell_fmt='.2f', cell_font_size=None, text_color_threshold=None) Plot a heatmap via matplotlib. Parameters conf_mat : array-like, shape = [n_rows, n_columns] And arbitrary 2D array. hide_spines : bool (default: False) Hides axis spines if True. hide_ticks : bool (default: False) Hides axis ticks if True figsize : tuple (default: (2.5, 2.5)) Height and width of the figure cmap : matplotlib colormap (default: None ) Uses matplotlib.pyplot.cm.viridis if None colorbar : bool (default: True) Shows a colorbar if True row_names : array-like, shape = [n_rows] (default: None) List of row names to be used as y-axis tick labels. column_names : array-like, shape = [n_columns] (default: None) List of column names to be used as x-axis tick labels. column_name_rotation : int (default: 45) Number of degrees for rotating column x-tick labels. cell_values : bool (default: True) Plots cell values if True. cell_fmt : string (default: '.2f') Format specification for cell values (if cell_values=True ) cell_font_size : int (default: None) Font size for cell values (if cell_values=True ) text_color_threshold : float (default: None) Threshold for the black/white text threshold of the text annotation. Default (None) tried to infer a good threshold automatically using np.max(normed_matrix) / 2 . Returns fig, ax : matplotlib.pyplot subplot objects Figure and axis elements of the subplot. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/plotting/heatmap/","title":"heatmap"},{"location":"api_modules/mlxtend.plotting/plot_confusion_matrix/","text":"plot_confusion_matrix plot_confusion_matrix(conf_mat, hide_spines=False, hide_ticks=False, figsize=None, cmap=None, colorbar=False, show_absolute=True, show_normed=False, norm_colormap=None, class_names=None, figure=None, axis=None, fontcolor_threshold=0.5) Plot a confusion matrix via matplotlib. Parameters conf_mat : array-like, shape = [n_classes, n_classes] Confusion matrix from evaluate.confusion matrix. hide_spines : bool (default: False) Hides axis spines if True. hide_ticks : bool (default: False) Hides axis ticks if True figsize : tuple (default: (2.5, 2.5)) Height and width of the figure cmap : matplotlib colormap (default: None ) Uses matplotlib.pyplot.cm.Blues if None colorbar : bool (default: False) Shows a colorbar if True show_absolute : bool (default: True) Shows absolute confusion matrix coefficients if True. At least one of show_absolute or show_normed must be True. show_normed : bool (default: False) Shows normed confusion matrix coefficients if True. The normed confusion matrix coefficients give the proportion of training examples per class that are assigned the correct label. At least one of show_absolute or show_normed must be True. norm_colormap : bool (default: False) Matplotlib color normalization object to normalize the color scale, e.g., matplotlib.colors.LogNorm() . class_names : array-like, shape = [n_classes] (default: None) List of class names. If not None , ticks will be set to these values. figure : None or Matplotlib figure (default: None) If None will create a new figure. axis : None or Matplotlib figure axis (default: None) If None will create a new axis. fontcolor_threshold : Float (default: 0.5) Sets a threshold for choosing black and white font colors for the cells. By default all values larger than 0.5 times the maximum cell value are converted to white, and everything equal or smaller than 0.5 times the maximum cell value are converted to black. Returns fig, ax : matplotlib.pyplot subplot objects Figure and axis elements of the subplot. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/plotting/plot_confusion_matrix/","title":"Plot confusion matrix"},{"location":"api_modules/mlxtend.plotting/plot_confusion_matrix/#plot_confusion_matrix","text":"plot_confusion_matrix(conf_mat, hide_spines=False, hide_ticks=False, figsize=None, cmap=None, colorbar=False, show_absolute=True, show_normed=False, norm_colormap=None, class_names=None, figure=None, axis=None, fontcolor_threshold=0.5) Plot a confusion matrix via matplotlib. Parameters conf_mat : array-like, shape = [n_classes, n_classes] Confusion matrix from evaluate.confusion matrix. hide_spines : bool (default: False) Hides axis spines if True. hide_ticks : bool (default: False) Hides axis ticks if True figsize : tuple (default: (2.5, 2.5)) Height and width of the figure cmap : matplotlib colormap (default: None ) Uses matplotlib.pyplot.cm.Blues if None colorbar : bool (default: False) Shows a colorbar if True show_absolute : bool (default: True) Shows absolute confusion matrix coefficients if True. At least one of show_absolute or show_normed must be True. show_normed : bool (default: False) Shows normed confusion matrix coefficients if True. The normed confusion matrix coefficients give the proportion of training examples per class that are assigned the correct label. At least one of show_absolute or show_normed must be True. norm_colormap : bool (default: False) Matplotlib color normalization object to normalize the color scale, e.g., matplotlib.colors.LogNorm() . class_names : array-like, shape = [n_classes] (default: None) List of class names. If not None , ticks will be set to these values. figure : None or Matplotlib figure (default: None) If None will create a new figure. axis : None or Matplotlib figure axis (default: None) If None will create a new axis. fontcolor_threshold : Float (default: 0.5) Sets a threshold for choosing black and white font colors for the cells. By default all values larger than 0.5 times the maximum cell value are converted to white, and everything equal or smaller than 0.5 times the maximum cell value are converted to black. Returns fig, ax : matplotlib.pyplot subplot objects Figure and axis elements of the subplot. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/plotting/plot_confusion_matrix/","title":"plot_confusion_matrix"},{"location":"api_modules/mlxtend.plotting/plot_decision_regions/","text":"plot_decision_regions plot_decision_regions(X, y, clf, feature_index=None, filler_feature_values=None, filler_feature_ranges=None, ax=None, X_highlight=None, zoom_factor=1.0, legend=1, hide_spines=True, markers='s^oxv<>', colors='#1f77b4,#ff7f0e,#3ca02c,#d62728,#9467bd,#8c564b,#e377c2,#7f7f7f,#bcbd22,#17becf', scatter_kwargs=None, contourf_kwargs=None, contour_kwargs=None, scatter_highlight_kwargs=None, n_jobs=None) Plot decision regions of a classifier. Please note that this functions assumes that class labels are labeled consecutively, e.g,. 0, 1, 2, 3, 4, and 5. If you have class labels with integer labels > 4, you may want to provide additional colors and/or markers as `colors` and `markers` arguments. See https://matplotlib.org/examples/color/named_colors.html for more information. Parameters X : array-like, shape = [n_samples, n_features] Feature Matrix. y : array-like, shape = [n_samples] True class labels. clf : Classifier object. Must have a .predict method. feature_index : array-like (default: (0,) for 1D, (0, 1) otherwise) Feature indices to use for plotting. The first index in feature_index will be on the x-axis, the second index will be on the y-axis. filler_feature_values : dict (default: None) Only needed for number features > 2. Dictionary of feature index-value pairs for the features not being plotted. filler_feature_ranges : dict (default: None) Only needed for number features > 2. Dictionary of feature index-value pairs for the features not being plotted. Will use the ranges provided to select training samples for plotting. ax : matplotlib.axes.Axes (default: None) An existing matplotlib Axes. Creates one if ax=None. X_highlight : array-like, shape = [n_samples, n_features] (default: None) An array with data points that are used to highlight samples in X . zoom_factor : float (default: 1.0) Controls the scale of the x- and y-axis of the decision plot. hide_spines : bool (default: True) Hide axis spines if True. legend : int (default: 1) Integer to specify the legend location. No legend if legend is 0. markers : str (default: 's^oxv<>') Scatterplot markers. colors : str (default: 'red,blue,limegreen,gray,cyan') Comma separated list of colors. scatter_kwargs : dict (default: None) Keyword arguments for underlying matplotlib scatter function. contourf_kwargs : dict (default: None) Keyword arguments for underlying matplotlib contourf function. contour_kwargs : dict (default: None) Keyword arguments for underlying matplotlib contour function (which draws the lines between decision regions). scatter_highlight_kwargs : dict (default: None) Keyword arguments for underlying matplotlib scatter function. n_jobs : int or None, optional (default=None) The number of CPUs to use to do the computation using Python's multiprocessing library. None means 1. -1 means using all processors. New in v0.22.0. Returns ax : matplotlib.axes.Axes object Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/plotting/plot_decision_regions/","title":"Plot decision regions"},{"location":"api_modules/mlxtend.plotting/plot_decision_regions/#plot_decision_regions","text":"plot_decision_regions(X, y, clf, feature_index=None, filler_feature_values=None, filler_feature_ranges=None, ax=None, X_highlight=None, zoom_factor=1.0, legend=1, hide_spines=True, markers='s^oxv<>', colors='#1f77b4,#ff7f0e,#3ca02c,#d62728,#9467bd,#8c564b,#e377c2,#7f7f7f,#bcbd22,#17becf', scatter_kwargs=None, contourf_kwargs=None, contour_kwargs=None, scatter_highlight_kwargs=None, n_jobs=None) Plot decision regions of a classifier. Please note that this functions assumes that class labels are labeled consecutively, e.g,. 0, 1, 2, 3, 4, and 5. If you have class labels with integer labels > 4, you may want to provide additional colors and/or markers as `colors` and `markers` arguments. See https://matplotlib.org/examples/color/named_colors.html for more information. Parameters X : array-like, shape = [n_samples, n_features] Feature Matrix. y : array-like, shape = [n_samples] True class labels. clf : Classifier object. Must have a .predict method. feature_index : array-like (default: (0,) for 1D, (0, 1) otherwise) Feature indices to use for plotting. The first index in feature_index will be on the x-axis, the second index will be on the y-axis. filler_feature_values : dict (default: None) Only needed for number features > 2. Dictionary of feature index-value pairs for the features not being plotted. filler_feature_ranges : dict (default: None) Only needed for number features > 2. Dictionary of feature index-value pairs for the features not being plotted. Will use the ranges provided to select training samples for plotting. ax : matplotlib.axes.Axes (default: None) An existing matplotlib Axes. Creates one if ax=None. X_highlight : array-like, shape = [n_samples, n_features] (default: None) An array with data points that are used to highlight samples in X . zoom_factor : float (default: 1.0) Controls the scale of the x- and y-axis of the decision plot. hide_spines : bool (default: True) Hide axis spines if True. legend : int (default: 1) Integer to specify the legend location. No legend if legend is 0. markers : str (default: 's^oxv<>') Scatterplot markers. colors : str (default: 'red,blue,limegreen,gray,cyan') Comma separated list of colors. scatter_kwargs : dict (default: None) Keyword arguments for underlying matplotlib scatter function. contourf_kwargs : dict (default: None) Keyword arguments for underlying matplotlib contourf function. contour_kwargs : dict (default: None) Keyword arguments for underlying matplotlib contour function (which draws the lines between decision regions). scatter_highlight_kwargs : dict (default: None) Keyword arguments for underlying matplotlib scatter function. n_jobs : int or None, optional (default=None) The number of CPUs to use to do the computation using Python's multiprocessing library. None means 1. -1 means using all processors. New in v0.22.0. Returns ax : matplotlib.axes.Axes object Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/plotting/plot_decision_regions/","title":"plot_decision_regions"},{"location":"api_modules/mlxtend.plotting/plot_learning_curves/","text":"plot_learning_curves plot_learning_curves(X_train, y_train, X_test, y_test, clf, train_marker='o', test_marker='^', scoring='misclassification error', suppress_plot=False, print_model=True, title_fontsize=12, style='default', legend_loc='best') Plots learning curves of a classifier. Parameters X_train : array-like, shape = [n_samples, n_features] Feature matrix of the training dataset. y_train : array-like, shape = [n_samples] True class labels of the training dataset. X_test : array-like, shape = [n_samples, n_features] Feature matrix of the test dataset. y_test : array-like, shape = [n_samples] True class labels of the test dataset. clf : Classifier object. Must have a .predict .fit method. train_marker : str (default: 'o') Marker for the training set line plot. test_marker : str (default: '^') Marker for the test set line plot. scoring : str (default: 'misclassification error') If not 'misclassification error', accepts the following metrics (from scikit-learn): {'accuracy', 'average_precision', 'f1_micro', 'f1_macro', 'f1_weighted', 'f1_samples', 'log_loss', 'precision', 'recall', 'roc_auc', 'adjusted_rand_score', 'mean_absolute_error', 'mean_squared_error', 'median_absolute_error', 'r2'} suppress_plot=False : bool (default: False) Suppress matplotlib plots if True. Recommended for testing purposes. print_model : bool (default: True) Print model parameters in plot title if True. title_fontsize : int (default: 12) Determines the size of the plot title font. style : str (default: 'default') Matplotlib style. For more styles, please see https://matplotlib.org/stable/gallery/style_sheets/style_sheets_reference.html legend_loc : str (default: 'best') Where to place the plot legend: {'best', 'upper left', 'upper right', 'lower left', 'lower right'} Returns errors : (training_error, test_error): tuple of lists Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/plotting/plot_learning_curves/","title":"Plot learning curves"},{"location":"api_modules/mlxtend.plotting/plot_learning_curves/#plot_learning_curves","text":"plot_learning_curves(X_train, y_train, X_test, y_test, clf, train_marker='o', test_marker='^', scoring='misclassification error', suppress_plot=False, print_model=True, title_fontsize=12, style='default', legend_loc='best') Plots learning curves of a classifier. Parameters X_train : array-like, shape = [n_samples, n_features] Feature matrix of the training dataset. y_train : array-like, shape = [n_samples] True class labels of the training dataset. X_test : array-like, shape = [n_samples, n_features] Feature matrix of the test dataset. y_test : array-like, shape = [n_samples] True class labels of the test dataset. clf : Classifier object. Must have a .predict .fit method. train_marker : str (default: 'o') Marker for the training set line plot. test_marker : str (default: '^') Marker for the test set line plot. scoring : str (default: 'misclassification error') If not 'misclassification error', accepts the following metrics (from scikit-learn): {'accuracy', 'average_precision', 'f1_micro', 'f1_macro', 'f1_weighted', 'f1_samples', 'log_loss', 'precision', 'recall', 'roc_auc', 'adjusted_rand_score', 'mean_absolute_error', 'mean_squared_error', 'median_absolute_error', 'r2'} suppress_plot=False : bool (default: False) Suppress matplotlib plots if True. Recommended for testing purposes. print_model : bool (default: True) Print model parameters in plot title if True. title_fontsize : int (default: 12) Determines the size of the plot title font. style : str (default: 'default') Matplotlib style. For more styles, please see https://matplotlib.org/stable/gallery/style_sheets/style_sheets_reference.html legend_loc : str (default: 'best') Where to place the plot legend: {'best', 'upper left', 'upper right', 'lower left', 'lower right'} Returns errors : (training_error, test_error): tuple of lists Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/plotting/plot_learning_curves/","title":"plot_learning_curves"},{"location":"api_modules/mlxtend.plotting/plot_linear_regression/","text":"plot_linear_regression plot_linear_regression(X, y, model=LinearRegression(), corr_func='pearsonr', scattercolor='blue', fit_style='k--', legend=True, xlim='auto') Plot a linear regression line fit. Parameters X : numpy array, shape = [n_samples,] Samples. y : numpy array, shape (n_samples,) Target values model: object (default: sklearn.linear_model.LinearRegression) Estimator object for regression. Must implement a .fit() and .predict() method. corr_func: str or function (default: 'pearsonr') Uses pearsonr from scipy.stats if corr_func='pearsonr'. to compute the regression slope. If not 'pearsonr', the corr_func , the corr_func parameter expects a function of the form func( , ) as inputs, which is expected to return a tuple (<correlation_coefficient>, <some_unused_value>) . scattercolor: string (default: blue) Color of scatter plot points. fit_style: string (default: k--) Style for the line fit. legend: bool (default: True) Plots legend with corr_coeff coef., fit coef., and intercept values. xlim: array-like (x_min, x_max) or 'auto' (default: 'auto') X-axis limits for the linear line fit. Returns regression_fit : tuple intercept, slope, corr_coeff (float, float, float) Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/plotting/plot_linear_regression/","title":"Plot linear regression"},{"location":"api_modules/mlxtend.plotting/plot_linear_regression/#plot_linear_regression","text":"plot_linear_regression(X, y, model=LinearRegression(), corr_func='pearsonr', scattercolor='blue', fit_style='k--', legend=True, xlim='auto') Plot a linear regression line fit. Parameters X : numpy array, shape = [n_samples,] Samples. y : numpy array, shape (n_samples,) Target values model: object (default: sklearn.linear_model.LinearRegression) Estimator object for regression. Must implement a .fit() and .predict() method. corr_func: str or function (default: 'pearsonr') Uses pearsonr from scipy.stats if corr_func='pearsonr'. to compute the regression slope. If not 'pearsonr', the corr_func , the corr_func parameter expects a function of the form func( , ) as inputs, which is expected to return a tuple (<correlation_coefficient>, <some_unused_value>) . scattercolor: string (default: blue) Color of scatter plot points. fit_style: string (default: k--) Style for the line fit. legend: bool (default: True) Plots legend with corr_coeff coef., fit coef., and intercept values. xlim: array-like (x_min, x_max) or 'auto' (default: 'auto') X-axis limits for the linear line fit. Returns regression_fit : tuple intercept, slope, corr_coeff (float, float, float) Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/plotting/plot_linear_regression/","title":"plot_linear_regression"},{"location":"api_modules/mlxtend.plotting/plot_pca_correlation_graph/","text":"plot_pca_correlation_graph plot_pca_correlation_graph(X, variables_names, dimensions=(1, 2), figure_axis_size=6, X_pca=None, explained_variance=None) Compute the PCA for X and plots the Correlation graph Parameters X : 2d array like. The columns represent the different variables and the rows are the samples of thos variables variables_names : array like Name of the columns (the variables) of X dimensions: tuple with two elements. dimensions to be plotted (x,y) figure_axis_size : size of the final frame. The figure created is a square with length and width equal to figure_axis_size. X_pca : np.ndarray, shape = [n_samples, n_components]. Optional. X_pca is the matrix of the transformed components from X. If not provided, the function computes PCA automatically using mlxtend.feature_extraction.PrincipalComponentAnalysis Expected n_componentes >= max(dimensions) explained_variance : 1 dimension np.ndarray, length = n_components Optional. explained_variance are the eigenvalues from the diagonalized covariance matrix on the PCA transformatiopn. If not provided, the function computes PCA independently Expected n_componentes == X.shape[1] Returns matplotlib_figure, correlation_matrix Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/plotting/plot_pca_correlation_graph/","title":"Plot pca correlation graph"},{"location":"api_modules/mlxtend.plotting/plot_pca_correlation_graph/#plot_pca_correlation_graph","text":"plot_pca_correlation_graph(X, variables_names, dimensions=(1, 2), figure_axis_size=6, X_pca=None, explained_variance=None) Compute the PCA for X and plots the Correlation graph Parameters X : 2d array like. The columns represent the different variables and the rows are the samples of thos variables variables_names : array like Name of the columns (the variables) of X dimensions: tuple with two elements. dimensions to be plotted (x,y) figure_axis_size : size of the final frame. The figure created is a square with length and width equal to figure_axis_size. X_pca : np.ndarray, shape = [n_samples, n_components]. Optional. X_pca is the matrix of the transformed components from X. If not provided, the function computes PCA automatically using mlxtend.feature_extraction.PrincipalComponentAnalysis Expected n_componentes >= max(dimensions) explained_variance : 1 dimension np.ndarray, length = n_components Optional. explained_variance are the eigenvalues from the diagonalized covariance matrix on the PCA transformatiopn. If not provided, the function computes PCA independently Expected n_componentes == X.shape[1] Returns matplotlib_figure, correlation_matrix Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/plotting/plot_pca_correlation_graph/","title":"plot_pca_correlation_graph"},{"location":"api_modules/mlxtend.plotting/plot_sequential_feature_selection/","text":"plot_sequential_feature_selection plot_sequential_feature_selection(metric_dict, figsize=None, kind='std_dev', color='blue', bcolor='steelblue', marker='o', alpha=0.2, ylabel='Performance', confidence_interval=0.95) Plot feature selection results. Parameters metric_dict : mlxtend.SequentialFeatureSelector.get_metric_dict() object figsize : tuple (default: None) Height and width of the figure kind : str (default: \"std_dev\") The kind of error bar or confidence interval in {'std_dev', 'std_err', 'ci', None}. color : str (default: \"blue\") Color of the lineplot (accepts any matplotlib color name) bcolor : str (default: \"steelblue\"). Color of the error bars / confidence intervals (accepts any matplotlib color name). marker : str (default: \"o\") Marker of the line plot (accepts any matplotlib marker name). alpha : float in [0, 1] (default: 0.2) Transparency of the error bars / confidence intervals. ylabel : str (default: \"Performance\") Y-axis label. confidence_interval : float (default: 0.95) Confidence level if kind='ci' . Returns fig : matplotlib.pyplot.figure() object Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/plotting/plot_sequential_feature_selection/","title":"Plot sequential feature selection"},{"location":"api_modules/mlxtend.plotting/plot_sequential_feature_selection/#plot_sequential_feature_selection","text":"plot_sequential_feature_selection(metric_dict, figsize=None, kind='std_dev', color='blue', bcolor='steelblue', marker='o', alpha=0.2, ylabel='Performance', confidence_interval=0.95) Plot feature selection results. Parameters metric_dict : mlxtend.SequentialFeatureSelector.get_metric_dict() object figsize : tuple (default: None) Height and width of the figure kind : str (default: \"std_dev\") The kind of error bar or confidence interval in {'std_dev', 'std_err', 'ci', None}. color : str (default: \"blue\") Color of the lineplot (accepts any matplotlib color name) bcolor : str (default: \"steelblue\"). Color of the error bars / confidence intervals (accepts any matplotlib color name). marker : str (default: \"o\") Marker of the line plot (accepts any matplotlib marker name). alpha : float in [0, 1] (default: 0.2) Transparency of the error bars / confidence intervals. ylabel : str (default: \"Performance\") Y-axis label. confidence_interval : float (default: 0.95) Confidence level if kind='ci' . Returns fig : matplotlib.pyplot.figure() object Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/plotting/plot_sequential_feature_selection/","title":"plot_sequential_feature_selection"},{"location":"api_modules/mlxtend.plotting/remove_borders/","text":"remove_borders remove_borders(axes, left=False, bottom=False, right=True, top=True) Remove chart junk from matplotlib plots. Parameters axes : iterable An iterable containing plt.gca() or plt.subplot() objects, e.g. [plt.gca()]. left : bool (default: False ) Hide left axis spine if True. bottom : bool (default: False ) Hide bottom axis spine if True. right : bool (default: True ) Hide right axis spine if True. top : bool (default: True ) Hide top axis spine if True. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/plotting/remove_chartjunk/","title":"Remove borders"},{"location":"api_modules/mlxtend.plotting/remove_borders/#remove_borders","text":"remove_borders(axes, left=False, bottom=False, right=True, top=True) Remove chart junk from matplotlib plots. Parameters axes : iterable An iterable containing plt.gca() or plt.subplot() objects, e.g. [plt.gca()]. left : bool (default: False ) Hide left axis spine if True. bottom : bool (default: False ) Hide bottom axis spine if True. right : bool (default: True ) Hide right axis spine if True. top : bool (default: True ) Hide top axis spine if True. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/plotting/remove_chartjunk/","title":"remove_borders"},{"location":"api_modules/mlxtend.plotting/scatter_hist/","text":"scatter_hist scatter_hist(x, y, xlabel=None, ylabel=None, figsize=(5, 5)) Scatter plot and individual feature histograms along axes. Parameters x : 1D array-like or Pandas Series X-axis values. y : 1D array-like or Pandas Series Y-axis values. xlabel : str (default: None ) Label for the X-axis values. If x is a pandas Series, and xlabel is None , the label is inferred automatically. ylabel : str (default: None ) Label for the X-axis values. If y is a pandas Series, and ylabel is None , the label is inferred automatically. figsize : tuple (default: (5, 5) ) Matplotlib figure size. Returns plot : Matplotlib Figure object","title":"Scatter hist"},{"location":"api_modules/mlxtend.plotting/scatter_hist/#scatter_hist","text":"scatter_hist(x, y, xlabel=None, ylabel=None, figsize=(5, 5)) Scatter plot and individual feature histograms along axes. Parameters x : 1D array-like or Pandas Series X-axis values. y : 1D array-like or Pandas Series Y-axis values. xlabel : str (default: None ) Label for the X-axis values. If x is a pandas Series, and xlabel is None , the label is inferred automatically. ylabel : str (default: None ) Label for the X-axis values. If y is a pandas Series, and ylabel is None , the label is inferred automatically. figsize : tuple (default: (5, 5) ) Matplotlib figure size. Returns plot : Matplotlib Figure object","title":"scatter_hist"},{"location":"api_modules/mlxtend.plotting/scatterplotmatrix/","text":"scatterplotmatrix scatterplotmatrix(X, fig_axes=None, names=None, figsize=(8, 8), alpha=1.0, kwargs) Lower triangular of a scatterplot matrix Parameters X : array-like, shape={num_examples, num_features} Design matrix containing data instances (examples) with multiple exploratory variables (features). fix_axes : tuple (default: None) A (fig, axes) tuple, where fig is an figure object and axes is an axes object created via matplotlib, for example, by calling the pyplot subplot function fig, axes = plt.subplots(...) names : list (default: None) A list of string names, which should have the same number of elements as there are features (columns) in X . figsize : tuple (default: (8, 8)) Height and width of the subplot grid. Ignored if fig_axes is not None . alpha : float (default: 1.0) Transparency for both the scatter plots and the histograms along the diagonal. **kwargs : kwargs Keyword arguments for the scatterplots. Returns fix_axes : tuple A (fig, axes) tuple, where fig is an figure object and axes is an axes object created via matplotlib, for example, by calling the pyplot subplot function fig, axes = plt.subplots(...) Examples For more usage examples, please see https://rasbt.github.io/mlxtend/user_guide/plotting/scatterplotmatrix/","title":"Scatterplotmatrix"},{"location":"api_modules/mlxtend.plotting/scatterplotmatrix/#scatterplotmatrix","text":"scatterplotmatrix(X, fig_axes=None, names=None, figsize=(8, 8), alpha=1.0, kwargs) Lower triangular of a scatterplot matrix Parameters X : array-like, shape={num_examples, num_features} Design matrix containing data instances (examples) with multiple exploratory variables (features). fix_axes : tuple (default: None) A (fig, axes) tuple, where fig is an figure object and axes is an axes object created via matplotlib, for example, by calling the pyplot subplot function fig, axes = plt.subplots(...) names : list (default: None) A list of string names, which should have the same number of elements as there are features (columns) in X . figsize : tuple (default: (8, 8)) Height and width of the subplot grid. Ignored if fig_axes is not None . alpha : float (default: 1.0) Transparency for both the scatter plots and the histograms along the diagonal. **kwargs : kwargs Keyword arguments for the scatterplots. Returns fix_axes : tuple A (fig, axes) tuple, where fig is an figure object and axes is an axes object created via matplotlib, for example, by calling the pyplot subplot function fig, axes = plt.subplots(...) Examples For more usage examples, please see https://rasbt.github.io/mlxtend/user_guide/plotting/scatterplotmatrix/","title":"scatterplotmatrix"},{"location":"api_modules/mlxtend.plotting/stacked_barplot/","text":"stacked_barplot stacked_barplot(df, bar_width='auto', colors='bgrcky', labels='index', rotation=90, legend_loc='best') Function to plot stacked barplots Parameters df : pandas.DataFrame A pandas DataFrame where the index denotes the x-axis labels, and the columns contain the different measurements for each row. bar_width: 'auto' or float (default: 'auto') Parameter to set the widths of the bars. if 'auto', the width is automatically determined by the number of columns in the dataset. colors: str (default: 'bgrcky') The colors of the bars. labels: 'index' or iterable (default: 'index') If 'index', the DataFrame index will be used as x-tick labels. rotation: int (default: 90) Parameter to rotate the x-axis labels. legend_loc : str (default: 'best') Location of the plot legend {best, upper left, upper right, lower left, lower right} No legend if legend_loc=False Returns fig : matplotlib.pyplot figure object Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/plotting/stacked_barplot/","title":"Stacked barplot"},{"location":"api_modules/mlxtend.plotting/stacked_barplot/#stacked_barplot","text":"stacked_barplot(df, bar_width='auto', colors='bgrcky', labels='index', rotation=90, legend_loc='best') Function to plot stacked barplots Parameters df : pandas.DataFrame A pandas DataFrame where the index denotes the x-axis labels, and the columns contain the different measurements for each row. bar_width: 'auto' or float (default: 'auto') Parameter to set the widths of the bars. if 'auto', the width is automatically determined by the number of columns in the dataset. colors: str (default: 'bgrcky') The colors of the bars. labels: 'index' or iterable (default: 'index') If 'index', the DataFrame index will be used as x-tick labels. rotation: int (default: 90) Parameter to rotate the x-axis labels. legend_loc : str (default: 'best') Location of the plot legend {best, upper left, upper right, lower left, lower right} No legend if legend_loc=False Returns fig : matplotlib.pyplot figure object Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/plotting/stacked_barplot/","title":"stacked_barplot"},{"location":"api_modules/mlxtend.preprocessing/CopyTransformer/","text":"CopyTransformer CopyTransformer() Transformer that returns a copy of the input array For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/preprocessing/CopyTransformer/ Methods fit(X, y=None) Mock method. Does nothing. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] (default: None) Returns self fit_transform(X, y=None) Return a copy of the input array. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] (default: None) Returns X_copy : copy of the input X array. get_params(deep=True) Get parameters for this estimator. Parameters deep : bool, default=True If True, will return the parameters for this estimator and contained subobjects that are estimators. Returns params : dict Parameter names mapped to their values. set_params( params) Set the parameters of this estimator. The method works on simple estimators as well as on nested objects (such as :class:`~sklearn.pipeline.Pipeline`). The latter have parameters of the form ``<component>__<parameter>`` so that it's possible to update each component of a nested object. Parameters **params : dict Estimator parameters. Returns self : estimator instance Estimator instance. transform(X, y=None) Return a copy of the input array. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] (default: None) Returns X_copy : copy of the input X array.","title":"CopyTransformer"},{"location":"api_modules/mlxtend.preprocessing/CopyTransformer/#copytransformer","text":"CopyTransformer() Transformer that returns a copy of the input array For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/preprocessing/CopyTransformer/","title":"CopyTransformer"},{"location":"api_modules/mlxtend.preprocessing/CopyTransformer/#methods","text":"fit(X, y=None) Mock method. Does nothing. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] (default: None) Returns self fit_transform(X, y=None) Return a copy of the input array. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] (default: None) Returns X_copy : copy of the input X array. get_params(deep=True) Get parameters for this estimator. Parameters deep : bool, default=True If True, will return the parameters for this estimator and contained subobjects that are estimators. Returns params : dict Parameter names mapped to their values. set_params( params) Set the parameters of this estimator. The method works on simple estimators as well as on nested objects (such as :class:`~sklearn.pipeline.Pipeline`). The latter have parameters of the form ``<component>__<parameter>`` so that it's possible to update each component of a nested object. Parameters **params : dict Estimator parameters. Returns self : estimator instance Estimator instance. transform(X, y=None) Return a copy of the input array. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] (default: None) Returns X_copy : copy of the input X array.","title":"Methods"},{"location":"api_modules/mlxtend.preprocessing/DenseTransformer/","text":"DenseTransformer DenseTransformer(return_copy=True) Convert a sparse array into a dense array. For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/preprocessing/DenseTransformer/ Methods fit(X, y=None) Mock method. Does nothing. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] (default: None) Returns self fit_transform(X, y=None) Return a dense version of the input array. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] (default: None) Returns X_dense : dense version of the input X array. get_params(deep=True) Get parameters for this estimator. Parameters deep : bool, default=True If True, will return the parameters for this estimator and contained subobjects that are estimators. Returns params : dict Parameter names mapped to their values. set_params( params) Set the parameters of this estimator. The method works on simple estimators as well as on nested objects (such as :class:`~sklearn.pipeline.Pipeline`). The latter have parameters of the form ``<component>__<parameter>`` so that it's possible to update each component of a nested object. Parameters **params : dict Estimator parameters. Returns self : estimator instance Estimator instance. transform(X, y=None) Return a dense version of the input array. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] (default: None) Returns X_dense : dense version of the input X array.","title":"DenseTransformer"},{"location":"api_modules/mlxtend.preprocessing/DenseTransformer/#densetransformer","text":"DenseTransformer(return_copy=True) Convert a sparse array into a dense array. For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/preprocessing/DenseTransformer/","title":"DenseTransformer"},{"location":"api_modules/mlxtend.preprocessing/DenseTransformer/#methods","text":"fit(X, y=None) Mock method. Does nothing. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] (default: None) Returns self fit_transform(X, y=None) Return a dense version of the input array. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] (default: None) Returns X_dense : dense version of the input X array. get_params(deep=True) Get parameters for this estimator. Parameters deep : bool, default=True If True, will return the parameters for this estimator and contained subobjects that are estimators. Returns params : dict Parameter names mapped to their values. set_params( params) Set the parameters of this estimator. The method works on simple estimators as well as on nested objects (such as :class:`~sklearn.pipeline.Pipeline`). The latter have parameters of the form ``<component>__<parameter>`` so that it's possible to update each component of a nested object. Parameters **params : dict Estimator parameters. Returns self : estimator instance Estimator instance. transform(X, y=None) Return a dense version of the input array. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] (default: None) Returns X_dense : dense version of the input X array.","title":"Methods"},{"location":"api_modules/mlxtend.preprocessing/MeanCenterer/","text":"MeanCenterer MeanCenterer() Column centering of vectors and matrices. Attributes col_means : numpy.ndarray [n_columns] NumPy array storing the mean values for centering after fitting the MeanCenterer object. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/preprocessing/MeanCenterer/ Methods fit(X) Gets the column means for mean centering. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Array of data vectors, where n_samples is the number of samples and n_features is the number of features. Returns self fit_transform(X) Fits and transforms an arry. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Array of data vectors, where n_samples is the number of samples and n_features is the number of features. Returns X_tr : {array-like, sparse matrix}, shape = [n_samples, n_features] A copy of the input array with the columns centered. transform(X) Centers a NumPy array. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Array of data vectors, where n_samples is the number of samples and n_features is the number of features. Returns X_tr : {array-like, sparse matrix}, shape = [n_samples, n_features] A copy of the input array with the columns centered.","title":"MeanCenterer"},{"location":"api_modules/mlxtend.preprocessing/MeanCenterer/#meancenterer","text":"MeanCenterer() Column centering of vectors and matrices. Attributes col_means : numpy.ndarray [n_columns] NumPy array storing the mean values for centering after fitting the MeanCenterer object. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/preprocessing/MeanCenterer/","title":"MeanCenterer"},{"location":"api_modules/mlxtend.preprocessing/MeanCenterer/#methods","text":"fit(X) Gets the column means for mean centering. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Array of data vectors, where n_samples is the number of samples and n_features is the number of features. Returns self fit_transform(X) Fits and transforms an arry. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Array of data vectors, where n_samples is the number of samples and n_features is the number of features. Returns X_tr : {array-like, sparse matrix}, shape = [n_samples, n_features] A copy of the input array with the columns centered. transform(X) Centers a NumPy array. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Array of data vectors, where n_samples is the number of samples and n_features is the number of features. Returns X_tr : {array-like, sparse matrix}, shape = [n_samples, n_features] A copy of the input array with the columns centered.","title":"Methods"},{"location":"api_modules/mlxtend.preprocessing/TransactionEncoder/","text":"TransactionEncoder TransactionEncoder() Encoder class for transaction data in Python lists Parameters None Attributes columns_: list List of unique names in the X input list of lists Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/preprocessing/TransactionEncoder/ Methods fit(X) Learn unique column names from transaction DataFrame Parameters X : list of lists A python list of lists, where the outer list stores the n transactions and the inner list stores the items in each transaction. For example, [['Apple', 'Beer', 'Rice', 'Chicken'], ['Apple', 'Beer', 'Rice'], ['Apple', 'Beer'], ['Apple', 'Bananas'], ['Milk', 'Beer', 'Rice', 'Chicken'], ['Milk', 'Beer', 'Rice'], ['Milk', 'Beer'], ['Apple', 'Bananas']] fit_transform(X, sparse=False) Fit a TransactionEncoder encoder and transform a dataset. get_params(deep=True) Get parameters for this estimator. Parameters deep : bool, default=True If True, will return the parameters for this estimator and contained subobjects that are estimators. Returns params : dict Parameter names mapped to their values. inverse_transform(array) Transforms an encoded NumPy array back into transactions. Parameters array : NumPy array [n_transactions, n_unique_items] The NumPy one-hot encoded boolean array of the input transactions, where the columns represent the unique items found in the input array in alphabetic order For example, array([[True , False, True , True , False, True ], [True , False, True , False, False, True ], [True , False, True , False, False, False], [True , True , False, False, False, False], [False, False, True , True , True , True ], [False, False, True , False, True , True ], [False, False, True , False, True , False], [True , True , False, False, False, False]]) The corresponding column labels are available as self.columns_, e.g., ['Apple', 'Bananas', 'Beer', 'Chicken', 'Milk', 'Rice'] Returns X : list of lists A python list of lists, where the outer list stores the n transactions and the inner list stores the items in each transaction. For example, [['Apple', 'Beer', 'Rice', 'Chicken'], ['Apple', 'Beer', 'Rice'], ['Apple', 'Beer'], ['Apple', 'Bananas'], ['Milk', 'Beer', 'Rice', 'Chicken'], ['Milk', 'Beer', 'Rice'], ['Milk', 'Beer'], ['Apple', 'Bananas']] set_output( , transform=None)* Set output container. See :ref:`sphx_glr_auto_examples_miscellaneous_plot_set_output.py` for an example on how to use the API. Parameters transform : {\"default\", \"pandas\"}, default=None Configure output of transform and fit_transform . \"default\" : Default output format of a transformer \"pandas\" : DataFrame output None : Transform configuration is unchanged Returns self : estimator instance Estimator instance. set_params( params) Set the parameters of this estimator. The method works on simple estimators as well as on nested objects (such as :class:`~sklearn.pipeline.Pipeline`). The latter have parameters of the form ``<component>__<parameter>`` so that it's possible to update each component of a nested object. Parameters **params : dict Estimator parameters. Returns self : estimator instance Estimator instance. transform(X, sparse=False) Transform transactions into a one-hot encoded NumPy array. Parameters X : list of lists A python list of lists, where the outer list stores the n transactions and the inner list stores the items in each transaction. For example, [['Apple', 'Beer', 'Rice', 'Chicken'], ['Apple', 'Beer', 'Rice'], ['Apple', 'Beer'], ['Apple', 'Bananas'], ['Milk', 'Beer', 'Rice', 'Chicken'], ['Milk', 'Beer', 'Rice'], ['Milk', 'Beer'], ['Apple', 'Bananas']] sparse: bool (default=False) If True, transform will return Compressed Sparse Row matrix instead of the regular one. Returns array : NumPy array [n_transactions, n_unique_items] if sparse=False (default). Compressed Sparse Row matrix otherwise The one-hot encoded boolean array of the input transactions, where the columns represent the unique items found in the input array in alphabetic order. Exact representation depends on the sparse argument For example, array([[True , False, True , True , False, True ], [True , False, True , False, False, True ], [True , False, True , False, False, False], [True , True , False, False, False, False], [False, False, True , True , True , True ], [False, False, True , False, True , True ], [False, False, True , False, True , False], [True , True , False, False, False, False]]) The corresponding column labels are available as self.columns_, e.g., ['Apple', 'Bananas', 'Beer', 'Chicken', 'Milk', 'Rice']","title":"TransactionEncoder"},{"location":"api_modules/mlxtend.preprocessing/TransactionEncoder/#transactionencoder","text":"TransactionEncoder() Encoder class for transaction data in Python lists Parameters None Attributes columns_: list List of unique names in the X input list of lists Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/preprocessing/TransactionEncoder/","title":"TransactionEncoder"},{"location":"api_modules/mlxtend.preprocessing/TransactionEncoder/#methods","text":"fit(X) Learn unique column names from transaction DataFrame Parameters X : list of lists A python list of lists, where the outer list stores the n transactions and the inner list stores the items in each transaction. For example, [['Apple', 'Beer', 'Rice', 'Chicken'], ['Apple', 'Beer', 'Rice'], ['Apple', 'Beer'], ['Apple', 'Bananas'], ['Milk', 'Beer', 'Rice', 'Chicken'], ['Milk', 'Beer', 'Rice'], ['Milk', 'Beer'], ['Apple', 'Bananas']] fit_transform(X, sparse=False) Fit a TransactionEncoder encoder and transform a dataset. get_params(deep=True) Get parameters for this estimator. Parameters deep : bool, default=True If True, will return the parameters for this estimator and contained subobjects that are estimators. Returns params : dict Parameter names mapped to their values. inverse_transform(array) Transforms an encoded NumPy array back into transactions. Parameters array : NumPy array [n_transactions, n_unique_items] The NumPy one-hot encoded boolean array of the input transactions, where the columns represent the unique items found in the input array in alphabetic order For example, array([[True , False, True , True , False, True ], [True , False, True , False, False, True ], [True , False, True , False, False, False], [True , True , False, False, False, False], [False, False, True , True , True , True ], [False, False, True , False, True , True ], [False, False, True , False, True , False], [True , True , False, False, False, False]]) The corresponding column labels are available as self.columns_, e.g., ['Apple', 'Bananas', 'Beer', 'Chicken', 'Milk', 'Rice'] Returns X : list of lists A python list of lists, where the outer list stores the n transactions and the inner list stores the items in each transaction. For example, [['Apple', 'Beer', 'Rice', 'Chicken'], ['Apple', 'Beer', 'Rice'], ['Apple', 'Beer'], ['Apple', 'Bananas'], ['Milk', 'Beer', 'Rice', 'Chicken'], ['Milk', 'Beer', 'Rice'], ['Milk', 'Beer'], ['Apple', 'Bananas']] set_output( , transform=None)* Set output container. See :ref:`sphx_glr_auto_examples_miscellaneous_plot_set_output.py` for an example on how to use the API. Parameters transform : {\"default\", \"pandas\"}, default=None Configure output of transform and fit_transform . \"default\" : Default output format of a transformer \"pandas\" : DataFrame output None : Transform configuration is unchanged Returns self : estimator instance Estimator instance. set_params( params) Set the parameters of this estimator. The method works on simple estimators as well as on nested objects (such as :class:`~sklearn.pipeline.Pipeline`). The latter have parameters of the form ``<component>__<parameter>`` so that it's possible to update each component of a nested object. Parameters **params : dict Estimator parameters. Returns self : estimator instance Estimator instance. transform(X, sparse=False) Transform transactions into a one-hot encoded NumPy array. Parameters X : list of lists A python list of lists, where the outer list stores the n transactions and the inner list stores the items in each transaction. For example, [['Apple', 'Beer', 'Rice', 'Chicken'], ['Apple', 'Beer', 'Rice'], ['Apple', 'Beer'], ['Apple', 'Bananas'], ['Milk', 'Beer', 'Rice', 'Chicken'], ['Milk', 'Beer', 'Rice'], ['Milk', 'Beer'], ['Apple', 'Bananas']] sparse: bool (default=False) If True, transform will return Compressed Sparse Row matrix instead of the regular one. Returns array : NumPy array [n_transactions, n_unique_items] if sparse=False (default). Compressed Sparse Row matrix otherwise The one-hot encoded boolean array of the input transactions, where the columns represent the unique items found in the input array in alphabetic order. Exact representation depends on the sparse argument For example, array([[True , False, True , True , False, True ], [True , False, True , False, False, True ], [True , False, True , False, False, False], [True , True , False, False, False, False], [False, False, True , True , True , True ], [False, False, True , False, True , True ], [False, False, True , False, True , False], [True , True , False, False, False, False]]) The corresponding column labels are available as self.columns_, e.g., ['Apple', 'Bananas', 'Beer', 'Chicken', 'Milk', 'Rice']","title":"Methods"},{"location":"api_modules/mlxtend.preprocessing/minmax_scaling/","text":"minmax_scaling minmax_scaling(array, columns, min_val=0, max_val=1) Min max scaling of pandas' DataFrames. Parameters array : pandas DataFrame or NumPy ndarray, shape = [n_rows, n_columns]. columns : array-like, shape = [n_columns] Array-like with column names, e.g., ['col1', 'col2', ...] or column indices [0, 2, 4, ...] min_val : int or float , optional (default= 0 ) minimum value after rescaling. max_val : int or float , optional (default= 1 ) maximum value after rescaling. Returns df_new : pandas DataFrame object. Copy of the array or DataFrame with rescaled columns. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/preprocessing/minmax_scaling/","title":"Minmax scaling"},{"location":"api_modules/mlxtend.preprocessing/minmax_scaling/#minmax_scaling","text":"minmax_scaling(array, columns, min_val=0, max_val=1) Min max scaling of pandas' DataFrames. Parameters array : pandas DataFrame or NumPy ndarray, shape = [n_rows, n_columns]. columns : array-like, shape = [n_columns] Array-like with column names, e.g., ['col1', 'col2', ...] or column indices [0, 2, 4, ...] min_val : int or float , optional (default= 0 ) minimum value after rescaling. max_val : int or float , optional (default= 1 ) maximum value after rescaling. Returns df_new : pandas DataFrame object. Copy of the array or DataFrame with rescaled columns. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/preprocessing/minmax_scaling/","title":"minmax_scaling"},{"location":"api_modules/mlxtend.preprocessing/one_hot/","text":"one_hot one_hot(y, num_labels='auto', dtype='float') One-hot encoding of class labels Parameters y : array-like, shape = [n_classlabels] Python list or numpy array consisting of class labels. num_labels : int or 'auto' Number of unique labels in the class label array. Infers the number of unique labels from the input array if set to 'auto'. dtype : str NumPy array type (float, float32, float64) of the output array. Returns ary : numpy.ndarray, shape = [n_classlabels] One-hot encoded array, where each sample is represented as a row vector in the returned array. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/preprocessing/one_hot/","title":"One hot"},{"location":"api_modules/mlxtend.preprocessing/one_hot/#one_hot","text":"one_hot(y, num_labels='auto', dtype='float') One-hot encoding of class labels Parameters y : array-like, shape = [n_classlabels] Python list or numpy array consisting of class labels. num_labels : int or 'auto' Number of unique labels in the class label array. Infers the number of unique labels from the input array if set to 'auto'. dtype : str NumPy array type (float, float32, float64) of the output array. Returns ary : numpy.ndarray, shape = [n_classlabels] One-hot encoded array, where each sample is represented as a row vector in the returned array. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/preprocessing/one_hot/","title":"one_hot"},{"location":"api_modules/mlxtend.preprocessing/shuffle_arrays_unison/","text":"shuffle_arrays_unison shuffle_arrays_unison(arrays, random_seed=None) Shuffle NumPy arrays in unison. Parameters arrays : array-like, shape = [n_arrays] A list of NumPy arrays. random_seed : int (default: None) Sets the random state. Returns shuffled_arrays : A list of NumPy arrays after shuffling. Examples ``` >>> import numpy as np >>> from mlxtend.preprocessing import shuffle_arrays_unison >>> X1 = np.array([[1, 2, 3], [4, 5, 6], [7, 8, 9]]) >>> y1 = np.array([1, 2, 3]) >>> X2, y2 = shuffle_arrays_unison(arrays=[X1, y1], random_seed=3) >>> assert(X2.all() == np.array([[4, 5, 6], [1, 2, 3], [7, 8, 9]]).all()) >>> assert(y2.all() == np.array([2, 1, 3]).all()) >>> For more usage examples, please see https://rasbt.github.io/mlxtend/user_guide/preprocessing/shuffle_arrays_unison/ ```","title":"Shuffle arrays unison"},{"location":"api_modules/mlxtend.preprocessing/shuffle_arrays_unison/#shuffle_arrays_unison","text":"shuffle_arrays_unison(arrays, random_seed=None) Shuffle NumPy arrays in unison. Parameters arrays : array-like, shape = [n_arrays] A list of NumPy arrays. random_seed : int (default: None) Sets the random state. Returns shuffled_arrays : A list of NumPy arrays after shuffling. Examples ``` >>> import numpy as np >>> from mlxtend.preprocessing import shuffle_arrays_unison >>> X1 = np.array([[1, 2, 3], [4, 5, 6], [7, 8, 9]]) >>> y1 = np.array([1, 2, 3]) >>> X2, y2 = shuffle_arrays_unison(arrays=[X1, y1], random_seed=3) >>> assert(X2.all() == np.array([[4, 5, 6], [1, 2, 3], [7, 8, 9]]).all()) >>> assert(y2.all() == np.array([2, 1, 3]).all()) >>> For more usage examples, please see https://rasbt.github.io/mlxtend/user_guide/preprocessing/shuffle_arrays_unison/ ```","title":"shuffle_arrays_unison"},{"location":"api_modules/mlxtend.preprocessing/standardize/","text":"standardize standardize(array, columns=None, ddof=0, return_params=False, params=None) Standardize columns in pandas DataFrames. Parameters array : pandas DataFrame or NumPy ndarray, shape = [n_rows, n_columns]. columns : array-like, shape = [n_columns] (default: None) Array-like with column names, e.g., ['col1', 'col2', ...] or column indices [0, 2, 4, ...] If None, standardizes all columns. ddof : int (default: 0) Delta Degrees of Freedom. The divisor used in calculations is N - ddof, where N represents the number of elements. return_params : dict (default: False) If set to True, a dictionary is returned in addition to the standardized array. The parameter dictionary contains the column means ('avgs') and standard deviations ('stds') of the individual columns. params : dict (default: None) A dictionary with column means and standard deviations as returned by the standardize function if return_params was set to True. If a params dictionary is provided, the standardize function will use these instead of computing them from the current array. Notes If all values in a given column are the same, these values are all set to 0.0 . The standard deviation in the parameters dictionary is consequently set to 1.0 to avoid dividing by zero. Returns df_new : pandas DataFrame object. Copy of the array or DataFrame with standardized columns. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/preprocessing/standardize/","title":"Standardize"},{"location":"api_modules/mlxtend.preprocessing/standardize/#standardize","text":"standardize(array, columns=None, ddof=0, return_params=False, params=None) Standardize columns in pandas DataFrames. Parameters array : pandas DataFrame or NumPy ndarray, shape = [n_rows, n_columns]. columns : array-like, shape = [n_columns] (default: None) Array-like with column names, e.g., ['col1', 'col2', ...] or column indices [0, 2, 4, ...] If None, standardizes all columns. ddof : int (default: 0) Delta Degrees of Freedom. The divisor used in calculations is N - ddof, where N represents the number of elements. return_params : dict (default: False) If set to True, a dictionary is returned in addition to the standardized array. The parameter dictionary contains the column means ('avgs') and standard deviations ('stds') of the individual columns. params : dict (default: None) A dictionary with column means and standard deviations as returned by the standardize function if return_params was set to True. If a params dictionary is provided, the standardize function will use these instead of computing them from the current array. Notes If all values in a given column are the same, these values are all set to 0.0 . The standard deviation in the parameters dictionary is consequently set to 1.0 to avoid dividing by zero. Returns df_new : pandas DataFrame object. Copy of the array or DataFrame with standardized columns. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/preprocessing/standardize/","title":"standardize"},{"location":"api_modules/mlxtend.regressor/LinearRegression/","text":"LinearRegression LinearRegression(method='direct', eta=0.01, epochs=50, minibatches=None, random_seed=None, print_progress=0) Ordinary least squares linear regression. Parameters method : string (default: 'direct') For gradient descent-based optimization, use sgd (see minibatch parameter for further options). Otherwise, if direct (default), the analytical method is used. For alternative, numerically more stable solutions, use either qr (QR decomopisition) or svd (Singular Value Decomposition). eta : float (default: 0.01) solver learning rate (between 0.0 and 1.0). Used with method = 'sgd' . (See methods parameter for details) epochs : int (default: 50) Passes over the training dataset. Prior to each epoch, the dataset is shuffled if minibatches > 1 to prevent cycles in stochastic gradient descent. Used with method = 'sgd' . (See methods parameter for details) minibatches : int (default: None) The number of minibatches for gradient-based optimization. If None: Direct method, QR, or SVD method (see method parameter for details) If 1: Gradient Descent learning If len(y): Stochastic Gradient Descent learning If 1 < minibatches < len(y): Minibatch learning random_seed : int (default: None) Set random state for shuffling and initializing the weights. Used in method = 'sgd' . (See methods parameter for details) print_progress : int (default: 0) Prints progress in fitting to stderr if method = 'sgd' . 0: No output 1: Epochs elapsed and cost 2: 1 plus time elapsed 3: 2 plus estimated time until completion Attributes w_ : 2d-array, shape={n_features, 1} Model weights after fitting. b_ : 1d-array, shape={1,} Bias unit after fitting. cost_ : list Sum of squared errors after each epoch; ignored if solver='normal equation' Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/regressor/LinearRegression/ Methods fit(X, y, init_params=True) Learn model from training data. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values. init_params : bool (default: True) Re-initializes model parameters prior to fitting. Set False to continue training with weights from a previous model fitting. Returns self : object get_params(deep=True) Get parameters for this estimator. Parameters deep : boolean, optional If True, will return the parameters for this estimator and contained subobjects that are estimators. Returns params : mapping of string to any Parameter names mapped to their values.' adapted from https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/base.py Author: Gael Varoquaux gael.varoquaux@normalesup.org License: BSD 3 clause predict(X) Predict targets from X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns target_values : array-like, shape = [n_samples] Predicted target values. set_params( params) Set the parameters of this estimator. The method works on simple estimators as well as on nested objects (such as pipelines). The latter have parameters of the form <component>__<parameter> so that it's possible to update each component of a nested object. Returns self adapted from https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/base.py Author: Gael Varoquaux <gael.varoquaux@normalesup.org> License: BSD 3 clause","title":"LinearRegression"},{"location":"api_modules/mlxtend.regressor/LinearRegression/#linearregression","text":"LinearRegression(method='direct', eta=0.01, epochs=50, minibatches=None, random_seed=None, print_progress=0) Ordinary least squares linear regression. Parameters method : string (default: 'direct') For gradient descent-based optimization, use sgd (see minibatch parameter for further options). Otherwise, if direct (default), the analytical method is used. For alternative, numerically more stable solutions, use either qr (QR decomopisition) or svd (Singular Value Decomposition). eta : float (default: 0.01) solver learning rate (between 0.0 and 1.0). Used with method = 'sgd' . (See methods parameter for details) epochs : int (default: 50) Passes over the training dataset. Prior to each epoch, the dataset is shuffled if minibatches > 1 to prevent cycles in stochastic gradient descent. Used with method = 'sgd' . (See methods parameter for details) minibatches : int (default: None) The number of minibatches for gradient-based optimization. If None: Direct method, QR, or SVD method (see method parameter for details) If 1: Gradient Descent learning If len(y): Stochastic Gradient Descent learning If 1 < minibatches < len(y): Minibatch learning random_seed : int (default: None) Set random state for shuffling and initializing the weights. Used in method = 'sgd' . (See methods parameter for details) print_progress : int (default: 0) Prints progress in fitting to stderr if method = 'sgd' . 0: No output 1: Epochs elapsed and cost 2: 1 plus time elapsed 3: 2 plus estimated time until completion Attributes w_ : 2d-array, shape={n_features, 1} Model weights after fitting. b_ : 1d-array, shape={1,} Bias unit after fitting. cost_ : list Sum of squared errors after each epoch; ignored if solver='normal equation' Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/regressor/LinearRegression/","title":"LinearRegression"},{"location":"api_modules/mlxtend.regressor/LinearRegression/#methods","text":"fit(X, y, init_params=True) Learn model from training data. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values. init_params : bool (default: True) Re-initializes model parameters prior to fitting. Set False to continue training with weights from a previous model fitting. Returns self : object get_params(deep=True) Get parameters for this estimator. Parameters deep : boolean, optional If True, will return the parameters for this estimator and contained subobjects that are estimators. Returns params : mapping of string to any Parameter names mapped to their values.' adapted from https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/base.py Author: Gael Varoquaux gael.varoquaux@normalesup.org License: BSD 3 clause predict(X) Predict targets from X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns target_values : array-like, shape = [n_samples] Predicted target values. set_params( params) Set the parameters of this estimator. The method works on simple estimators as well as on nested objects (such as pipelines). The latter have parameters of the form <component>__<parameter> so that it's possible to update each component of a nested object. Returns self adapted from https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/base.py Author: Gael Varoquaux <gael.varoquaux@normalesup.org> License: BSD 3 clause","title":"Methods"},{"location":"api_modules/mlxtend.regressor/StackingCVRegressor/","text":"StackingCVRegressor StackingCVRegressor(regressors, meta_regressor, cv=5, shuffle=True, random_state=None, verbose=0, refit=True, use_features_in_secondary=False, store_train_meta_features=False, n_jobs=None, pre_dispatch='2 n_jobs', multi_output=False)* A 'Stacking Cross-Validation' regressor for scikit-learn estimators. Parameters regressors : array-like, shape = [n_regressors] A list of regressors. Invoking the fit method on the StackingCVRegressor will fit clones of these original regressors that will be stored in the class attribute self.regr_ . meta_regressor : object The meta-regressor to be fitted on the ensemble of regressor cv : int, cross-validation generator or iterable, optional (default: 5) Determines the cross-validation splitting strategy. Possible inputs for cv are: - None, to use the default 5-fold cross validation, - integer, to specify the number of folds in a KFold , - An object to be used as a cross-validation generator. - An iterable yielding train, test splits. For integer/None inputs, it will use KFold cross-validation shuffle : bool (default: True) If True, and the cv argument is integer, the training data will be shuffled at fitting stage prior to cross-validation. If the cv argument is a specific cross validation technique, this argument is omitted. random_state : int, RandomState instance or None, optional (default: None) Constrols the randomness of the cv splitter. Used when cv is integer and shuffle=True . New in v0.16.0. verbose : int, optional (default=0) Controls the verbosity of the building process. New in v0.16.0 refit : bool (default: True) Clones the regressors for stacking regression if True (default) or else uses the original ones, which will be refitted on the dataset upon calling the fit method. Setting refit=False is recommended if you are working with estimators that are supporting the scikit-learn fit/predict API interface but are not compatible to scikit-learn's clone function. use_features_in_secondary : bool (default: False) If True, the meta-regressor will be trained both on the predictions of the original regressors and the original dataset. If False, the meta-regressor will be trained only on the predictions of the original regressors. store_train_meta_features : bool (default: False) If True, the meta-features computed from the training data used for fitting the meta-regressor stored in the self.train_meta_features_ array, which can be accessed after calling fit . n_jobs : int or None, optional (default=None) The number of CPUs to use to do the computation. None means 1 unless in a :obj: joblib.parallel_backend context. -1 means using all processors. See :term: Glossary <n_jobs> for more details. New in v0.16.0. pre_dispatch : int, or string, optional Controls the number of jobs that get dispatched during parallel execution. Reducing this number can be useful to avoid an explosion of memory consumption when more jobs get dispatched than CPUs can process. This parameter can be: - None, in which case all the jobs are immediately created and spawned. Use this for lightweight and fast-running jobs, to avoid delays due to on-demand spawning of the jobs - An int, giving the exact number of total jobs that are spawned - A string, giving an expression as a function of n_jobs, as in '2*n_jobs' multi_output : bool (default: False) If True, allow multi-output targets, but forbid nan or inf values. If False, y will be checked to be a vector. (New in v0.19.0.) Attributes train_meta_features : numpy array, shape = [n_samples, n_regressors] meta-features for training data, where n_samples is the number of samples in training data and len(self.regressors) is the number of regressors. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/regressor/StackingCVRegressor/ Methods fit(X, y, groups=None, sample_weight=None) Fit ensemble regressors and the meta-regressor. Parameters X : numpy array, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : numpy array, shape = [n_samples] or [n_samples, n_targets] Target values. Multiple targets are supported only if self.multi_output is True. groups : numpy array/None, shape = [n_samples] The group that each sample belongs to. This is used by specific folding strategies such as GroupKFold() sample_weight : array-like, shape = [n_samples], optional Sample weights passed as sample_weights to each regressor in the regressors list as well as the meta_regressor. Raises error if some regressor does not support sample_weight in the fit() method. Returns self : object fit_transform(X, y=None, fit_params) Fit to data, then transform it. Fits transformer to `X` and `y` with optional parameters `fit_params` and returns a transformed version of `X`. Parameters X : array-like of shape (n_samples, n_features) Input samples. y : array-like of shape (n_samples,) or (n_samples, n_outputs), default=None Target values (None for unsupervised transformations). **fit_params : dict Additional fit parameters. Returns X_new : ndarray array of shape (n_samples, n_features_new) Transformed array. get_params(deep=True) Get parameters for this estimator. Parameters deep : bool, default=True If True, will return the parameters for this estimator and contained subobjects that are estimators. Returns params : dict Parameter names mapped to their values. predict(X) Predict target values for X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns y_target : array-like, shape = [n_samples] or [n_samples, n_targets] Predicted target values. predict_meta_features(X) Get meta-features of test-data. Parameters X : numpy array, shape = [n_samples, n_features] Test vectors, where n_samples is the number of samples and n_features is the number of features. Returns meta-features : numpy array, shape = [n_samples, len(self.regressors)] meta-features for test data, where n_samples is the number of samples in test data and len(self.regressors) is the number of regressors. If self.multi_output is True, then the number of columns is len(self.regressors) * n_targets. score(X, y, sample_weight=None) Return the coefficient of determination of the prediction. The coefficient of determination :math:`R^2` is defined as :math:`(1 - \\frac{u}{v})`, where :math:`u` is the residual sum of squares ((y_true - y_pred)** 2).sum() and :math: v is the total sum of squares ((y_true - y_true.mean()) ** 2).sum() . The best possible score is 1.0 and it can be negative (because the model can be arbitrarily worse). A constant model that always predicts the expected value of y , disregarding the input features, would get a :math: R^2 score of 0.0. Parameters X : array-like of shape (n_samples, n_features) Test samples. For some estimators this may be a precomputed kernel matrix or a list of generic objects instead with shape (n_samples, n_samples_fitted) , where n_samples_fitted is the number of samples used in the fitting for the estimator. y : array-like of shape (n_samples,) or (n_samples, n_outputs) True values for X . sample_weight : array-like of shape (n_samples,), default=None Sample weights. Returns score : float :math: R^2 of self.predict(X) w.r.t. y . Notes The :math: R^2 score used when calling score on a regressor uses multioutput='uniform_average' from version 0.23 to keep consistent with default value of :func: ~sklearn.metrics.r2_score . This influences the score method of all the multioutput regressors (except for :class: ~sklearn.multioutput.MultiOutputRegressor ). set_output( , transform=None)* Set output container. See :ref:`sphx_glr_auto_examples_miscellaneous_plot_set_output.py` for an example on how to use the API. Parameters transform : {\"default\", \"pandas\"}, default=None Configure output of transform and fit_transform . \"default\" : Default output format of a transformer \"pandas\" : DataFrame output None : Transform configuration is unchanged Returns self : estimator instance Estimator instance. set_params( params) Set the parameters of this estimator. Valid parameter keys can be listed with ``get_params()``. Returns self Properties named_regressors Returns List of named estimator tuples, like [('svc', SVC(...))]","title":"StackingCVRegressor"},{"location":"api_modules/mlxtend.regressor/StackingCVRegressor/#stackingcvregressor","text":"StackingCVRegressor(regressors, meta_regressor, cv=5, shuffle=True, random_state=None, verbose=0, refit=True, use_features_in_secondary=False, store_train_meta_features=False, n_jobs=None, pre_dispatch='2 n_jobs', multi_output=False)* A 'Stacking Cross-Validation' regressor for scikit-learn estimators. Parameters regressors : array-like, shape = [n_regressors] A list of regressors. Invoking the fit method on the StackingCVRegressor will fit clones of these original regressors that will be stored in the class attribute self.regr_ . meta_regressor : object The meta-regressor to be fitted on the ensemble of regressor cv : int, cross-validation generator or iterable, optional (default: 5) Determines the cross-validation splitting strategy. Possible inputs for cv are: - None, to use the default 5-fold cross validation, - integer, to specify the number of folds in a KFold , - An object to be used as a cross-validation generator. - An iterable yielding train, test splits. For integer/None inputs, it will use KFold cross-validation shuffle : bool (default: True) If True, and the cv argument is integer, the training data will be shuffled at fitting stage prior to cross-validation. If the cv argument is a specific cross validation technique, this argument is omitted. random_state : int, RandomState instance or None, optional (default: None) Constrols the randomness of the cv splitter. Used when cv is integer and shuffle=True . New in v0.16.0. verbose : int, optional (default=0) Controls the verbosity of the building process. New in v0.16.0 refit : bool (default: True) Clones the regressors for stacking regression if True (default) or else uses the original ones, which will be refitted on the dataset upon calling the fit method. Setting refit=False is recommended if you are working with estimators that are supporting the scikit-learn fit/predict API interface but are not compatible to scikit-learn's clone function. use_features_in_secondary : bool (default: False) If True, the meta-regressor will be trained both on the predictions of the original regressors and the original dataset. If False, the meta-regressor will be trained only on the predictions of the original regressors. store_train_meta_features : bool (default: False) If True, the meta-features computed from the training data used for fitting the meta-regressor stored in the self.train_meta_features_ array, which can be accessed after calling fit . n_jobs : int or None, optional (default=None) The number of CPUs to use to do the computation. None means 1 unless in a :obj: joblib.parallel_backend context. -1 means using all processors. See :term: Glossary <n_jobs> for more details. New in v0.16.0. pre_dispatch : int, or string, optional Controls the number of jobs that get dispatched during parallel execution. Reducing this number can be useful to avoid an explosion of memory consumption when more jobs get dispatched than CPUs can process. This parameter can be: - None, in which case all the jobs are immediately created and spawned. Use this for lightweight and fast-running jobs, to avoid delays due to on-demand spawning of the jobs - An int, giving the exact number of total jobs that are spawned - A string, giving an expression as a function of n_jobs, as in '2*n_jobs' multi_output : bool (default: False) If True, allow multi-output targets, but forbid nan or inf values. If False, y will be checked to be a vector. (New in v0.19.0.) Attributes train_meta_features : numpy array, shape = [n_samples, n_regressors] meta-features for training data, where n_samples is the number of samples in training data and len(self.regressors) is the number of regressors. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/regressor/StackingCVRegressor/","title":"StackingCVRegressor"},{"location":"api_modules/mlxtend.regressor/StackingCVRegressor/#methods","text":"fit(X, y, groups=None, sample_weight=None) Fit ensemble regressors and the meta-regressor. Parameters X : numpy array, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : numpy array, shape = [n_samples] or [n_samples, n_targets] Target values. Multiple targets are supported only if self.multi_output is True. groups : numpy array/None, shape = [n_samples] The group that each sample belongs to. This is used by specific folding strategies such as GroupKFold() sample_weight : array-like, shape = [n_samples], optional Sample weights passed as sample_weights to each regressor in the regressors list as well as the meta_regressor. Raises error if some regressor does not support sample_weight in the fit() method. Returns self : object fit_transform(X, y=None, fit_params) Fit to data, then transform it. Fits transformer to `X` and `y` with optional parameters `fit_params` and returns a transformed version of `X`. Parameters X : array-like of shape (n_samples, n_features) Input samples. y : array-like of shape (n_samples,) or (n_samples, n_outputs), default=None Target values (None for unsupervised transformations). **fit_params : dict Additional fit parameters. Returns X_new : ndarray array of shape (n_samples, n_features_new) Transformed array. get_params(deep=True) Get parameters for this estimator. Parameters deep : bool, default=True If True, will return the parameters for this estimator and contained subobjects that are estimators. Returns params : dict Parameter names mapped to their values. predict(X) Predict target values for X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns y_target : array-like, shape = [n_samples] or [n_samples, n_targets] Predicted target values. predict_meta_features(X) Get meta-features of test-data. Parameters X : numpy array, shape = [n_samples, n_features] Test vectors, where n_samples is the number of samples and n_features is the number of features. Returns meta-features : numpy array, shape = [n_samples, len(self.regressors)] meta-features for test data, where n_samples is the number of samples in test data and len(self.regressors) is the number of regressors. If self.multi_output is True, then the number of columns is len(self.regressors) * n_targets. score(X, y, sample_weight=None) Return the coefficient of determination of the prediction. The coefficient of determination :math:`R^2` is defined as :math:`(1 - \\frac{u}{v})`, where :math:`u` is the residual sum of squares ((y_true - y_pred)** 2).sum() and :math: v is the total sum of squares ((y_true - y_true.mean()) ** 2).sum() . The best possible score is 1.0 and it can be negative (because the model can be arbitrarily worse). A constant model that always predicts the expected value of y , disregarding the input features, would get a :math: R^2 score of 0.0. Parameters X : array-like of shape (n_samples, n_features) Test samples. For some estimators this may be a precomputed kernel matrix or a list of generic objects instead with shape (n_samples, n_samples_fitted) , where n_samples_fitted is the number of samples used in the fitting for the estimator. y : array-like of shape (n_samples,) or (n_samples, n_outputs) True values for X . sample_weight : array-like of shape (n_samples,), default=None Sample weights. Returns score : float :math: R^2 of self.predict(X) w.r.t. y . Notes The :math: R^2 score used when calling score on a regressor uses multioutput='uniform_average' from version 0.23 to keep consistent with default value of :func: ~sklearn.metrics.r2_score . This influences the score method of all the multioutput regressors (except for :class: ~sklearn.multioutput.MultiOutputRegressor ). set_output( , transform=None)* Set output container. See :ref:`sphx_glr_auto_examples_miscellaneous_plot_set_output.py` for an example on how to use the API. Parameters transform : {\"default\", \"pandas\"}, default=None Configure output of transform and fit_transform . \"default\" : Default output format of a transformer \"pandas\" : DataFrame output None : Transform configuration is unchanged Returns self : estimator instance Estimator instance. set_params( params) Set the parameters of this estimator. Valid parameter keys can be listed with ``get_params()``. Returns self","title":"Methods"},{"location":"api_modules/mlxtend.regressor/StackingCVRegressor/#properties","text":"named_regressors Returns List of named estimator tuples, like [('svc', SVC(...))]","title":"Properties"},{"location":"api_modules/mlxtend.regressor/StackingRegressor/","text":"StackingRegressor StackingRegressor(regressors, meta_regressor, verbose=0, use_features_in_secondary=False, store_train_meta_features=False, refit=True, multi_output=False) A Stacking regressor for scikit-learn estimators for regression. Parameters regressors : array-like, shape = [n_regressors] A list of regressors. Invoking the fit method on the StackingRegressor will fit clones of those original regressors that will be stored in the class attribute self.regr_ . meta_regressor : object The meta-regressor to be fitted on the ensemble of regressors verbose : int, optional (default=0) Controls the verbosity of the building process. - verbose=0 (default): Prints nothing - verbose=1 : Prints the number & name of the regressor being fitted - verbose=2 : Prints info about the parameters of the regressor being fitted - verbose>2 : Changes verbose param of the underlying regressor to self.verbose - 2 use_features_in_secondary : bool (default: False) If True, the meta-regressor will be trained both on the predictions of the original regressors and the original dataset. If False, the meta-regressor will be trained only on the predictions of the original regressors. store_train_meta_features : bool (default: False) If True, the meta-features computed from the training data used for fitting the meta-regressor stored in the self.train_meta_features_ array, which can be accessed after calling fit . Attributes regr_ : list, shape=[n_regressors] Fitted regressors (clones of the original regressors) meta_regr_ : estimator Fitted meta-regressor (clone of the original meta-estimator) coef_ : array-like, shape = [n_features] Model coefficients of the fitted meta-estimator intercept_ : float Intercept of the fitted meta-estimator train_meta_features : numpy array, shape = [n_samples, len(self.regressors)] meta-features for training data, where n_samples is the number of samples in training data and len(self.regressors) is the number of regressors. refit : bool (default: True) Clones the regressors for stacking regression if True (default) or else uses the original ones, which will be refitted on the dataset upon calling the fit method. Setting refit=False is recommended if you are working with estimators that are supporting the scikit-learn fit/predict API interface but are not compatible to scikit-learn's clone function. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/regressor/StackingRegressor/ Methods fit(X, y, sample_weight=None) Learn weight coefficients from training data for each regressor. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : numpy array, shape = [n_samples] or [n_samples, n_targets] Target values. Multiple targets are supported only if self.multi_output is True. sample_weight : array-like, shape = [n_samples], optional Sample weights passed as sample_weights to each regressor in the regressors list as well as the meta_regressor. Raises error if some regressor does not support sample_weight in the fit() method. Returns self : object fit_transform(X, y=None, fit_params) Fit to data, then transform it. Fits transformer to `X` and `y` with optional parameters `fit_params` and returns a transformed version of `X`. Parameters X : array-like of shape (n_samples, n_features) Input samples. y : array-like of shape (n_samples,) or (n_samples, n_outputs), default=None Target values (None for unsupervised transformations). **fit_params : dict Additional fit parameters. Returns X_new : ndarray array of shape (n_samples, n_features_new) Transformed array. get_params(deep=True) Return estimator parameter names for GridSearch support. predict(X) Predict target values for X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns y_target : array-like, shape = [n_samples] or [n_samples, n_targets] Predicted target values. predict_meta_features(X) Get meta-features of test-data. Parameters X : numpy array, shape = [n_samples, n_features] Test vectors, where n_samples is the number of samples and n_features is the number of features. Returns meta-features : numpy array, shape = [n_samples, len(self.regressors)] meta-features for test data, where n_samples is the number of samples in test data and len(self.regressors) is the number of regressors. If self.multi_output is True, then the number of columns is len(self.regressors) * n_targets score(X, y, sample_weight=None) Return the coefficient of determination of the prediction. The coefficient of determination :math:`R^2` is defined as :math:`(1 - \\frac{u}{v})`, where :math:`u` is the residual sum of squares ((y_true - y_pred)** 2).sum() and :math: v is the total sum of squares ((y_true - y_true.mean()) ** 2).sum() . The best possible score is 1.0 and it can be negative (because the model can be arbitrarily worse). A constant model that always predicts the expected value of y , disregarding the input features, would get a :math: R^2 score of 0.0. Parameters X : array-like of shape (n_samples, n_features) Test samples. For some estimators this may be a precomputed kernel matrix or a list of generic objects instead with shape (n_samples, n_samples_fitted) , where n_samples_fitted is the number of samples used in the fitting for the estimator. y : array-like of shape (n_samples,) or (n_samples, n_outputs) True values for X . sample_weight : array-like of shape (n_samples,), default=None Sample weights. Returns score : float :math: R^2 of self.predict(X) w.r.t. y . Notes The :math: R^2 score used when calling score on a regressor uses multioutput='uniform_average' from version 0.23 to keep consistent with default value of :func: ~sklearn.metrics.r2_score . This influences the score method of all the multioutput regressors (except for :class: ~sklearn.multioutput.MultiOutputRegressor ). set_output( , transform=None)* Set output container. See :ref:`sphx_glr_auto_examples_miscellaneous_plot_set_output.py` for an example on how to use the API. Parameters transform : {\"default\", \"pandas\"}, default=None Configure output of transform and fit_transform . \"default\" : Default output format of a transformer \"pandas\" : DataFrame output None : Transform configuration is unchanged Returns self : estimator instance Estimator instance. set_params( params) Set the parameters of this estimator. Valid parameter keys can be listed with ``get_params()``. Returns self Properties coef_ None intercept_ None named_regressors None","title":"StackingRegressor"},{"location":"api_modules/mlxtend.regressor/StackingRegressor/#stackingregressor","text":"StackingRegressor(regressors, meta_regressor, verbose=0, use_features_in_secondary=False, store_train_meta_features=False, refit=True, multi_output=False) A Stacking regressor for scikit-learn estimators for regression. Parameters regressors : array-like, shape = [n_regressors] A list of regressors. Invoking the fit method on the StackingRegressor will fit clones of those original regressors that will be stored in the class attribute self.regr_ . meta_regressor : object The meta-regressor to be fitted on the ensemble of regressors verbose : int, optional (default=0) Controls the verbosity of the building process. - verbose=0 (default): Prints nothing - verbose=1 : Prints the number & name of the regressor being fitted - verbose=2 : Prints info about the parameters of the regressor being fitted - verbose>2 : Changes verbose param of the underlying regressor to self.verbose - 2 use_features_in_secondary : bool (default: False) If True, the meta-regressor will be trained both on the predictions of the original regressors and the original dataset. If False, the meta-regressor will be trained only on the predictions of the original regressors. store_train_meta_features : bool (default: False) If True, the meta-features computed from the training data used for fitting the meta-regressor stored in the self.train_meta_features_ array, which can be accessed after calling fit . Attributes regr_ : list, shape=[n_regressors] Fitted regressors (clones of the original regressors) meta_regr_ : estimator Fitted meta-regressor (clone of the original meta-estimator) coef_ : array-like, shape = [n_features] Model coefficients of the fitted meta-estimator intercept_ : float Intercept of the fitted meta-estimator train_meta_features : numpy array, shape = [n_samples, len(self.regressors)] meta-features for training data, where n_samples is the number of samples in training data and len(self.regressors) is the number of regressors. refit : bool (default: True) Clones the regressors for stacking regression if True (default) or else uses the original ones, which will be refitted on the dataset upon calling the fit method. Setting refit=False is recommended if you are working with estimators that are supporting the scikit-learn fit/predict API interface but are not compatible to scikit-learn's clone function. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/regressor/StackingRegressor/","title":"StackingRegressor"},{"location":"api_modules/mlxtend.regressor/StackingRegressor/#methods","text":"fit(X, y, sample_weight=None) Learn weight coefficients from training data for each regressor. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : numpy array, shape = [n_samples] or [n_samples, n_targets] Target values. Multiple targets are supported only if self.multi_output is True. sample_weight : array-like, shape = [n_samples], optional Sample weights passed as sample_weights to each regressor in the regressors list as well as the meta_regressor. Raises error if some regressor does not support sample_weight in the fit() method. Returns self : object fit_transform(X, y=None, fit_params) Fit to data, then transform it. Fits transformer to `X` and `y` with optional parameters `fit_params` and returns a transformed version of `X`. Parameters X : array-like of shape (n_samples, n_features) Input samples. y : array-like of shape (n_samples,) or (n_samples, n_outputs), default=None Target values (None for unsupervised transformations). **fit_params : dict Additional fit parameters. Returns X_new : ndarray array of shape (n_samples, n_features_new) Transformed array. get_params(deep=True) Return estimator parameter names for GridSearch support. predict(X) Predict target values for X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns y_target : array-like, shape = [n_samples] or [n_samples, n_targets] Predicted target values. predict_meta_features(X) Get meta-features of test-data. Parameters X : numpy array, shape = [n_samples, n_features] Test vectors, where n_samples is the number of samples and n_features is the number of features. Returns meta-features : numpy array, shape = [n_samples, len(self.regressors)] meta-features for test data, where n_samples is the number of samples in test data and len(self.regressors) is the number of regressors. If self.multi_output is True, then the number of columns is len(self.regressors) * n_targets score(X, y, sample_weight=None) Return the coefficient of determination of the prediction. The coefficient of determination :math:`R^2` is defined as :math:`(1 - \\frac{u}{v})`, where :math:`u` is the residual sum of squares ((y_true - y_pred)** 2).sum() and :math: v is the total sum of squares ((y_true - y_true.mean()) ** 2).sum() . The best possible score is 1.0 and it can be negative (because the model can be arbitrarily worse). A constant model that always predicts the expected value of y , disregarding the input features, would get a :math: R^2 score of 0.0. Parameters X : array-like of shape (n_samples, n_features) Test samples. For some estimators this may be a precomputed kernel matrix or a list of generic objects instead with shape (n_samples, n_samples_fitted) , where n_samples_fitted is the number of samples used in the fitting for the estimator. y : array-like of shape (n_samples,) or (n_samples, n_outputs) True values for X . sample_weight : array-like of shape (n_samples,), default=None Sample weights. Returns score : float :math: R^2 of self.predict(X) w.r.t. y . Notes The :math: R^2 score used when calling score on a regressor uses multioutput='uniform_average' from version 0.23 to keep consistent with default value of :func: ~sklearn.metrics.r2_score . This influences the score method of all the multioutput regressors (except for :class: ~sklearn.multioutput.MultiOutputRegressor ). set_output( , transform=None)* Set output container. See :ref:`sphx_glr_auto_examples_miscellaneous_plot_set_output.py` for an example on how to use the API. Parameters transform : {\"default\", \"pandas\"}, default=None Configure output of transform and fit_transform . \"default\" : Default output format of a transformer \"pandas\" : DataFrame output None : Transform configuration is unchanged Returns self : estimator instance Estimator instance. set_params( params) Set the parameters of this estimator. Valid parameter keys can be listed with ``get_params()``. Returns self","title":"Methods"},{"location":"api_modules/mlxtend.regressor/StackingRegressor/#properties","text":"coef_ None intercept_ None named_regressors None","title":"Properties"},{"location":"api_modules/mlxtend.text/generalize_names/","text":"generalize_names generalize_names(name, output_sep=' ', firstname_output_letters=1) Generalize a person's first and last name. Returns a person's name in the format <last_name><separator><firstname letter(s)> (all lowercase) Parameters name : str Name of the player output_sep : str (default: ' ') String for separating last name and first name in the output. firstname_output_letters : int Number of letters in the abbreviated first name. Returns gen_name : str The generalized name. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/text/generalize_names/","title":"Generalize names"},{"location":"api_modules/mlxtend.text/generalize_names/#generalize_names","text":"generalize_names(name, output_sep=' ', firstname_output_letters=1) Generalize a person's first and last name. Returns a person's name in the format <last_name><separator><firstname letter(s)> (all lowercase) Parameters name : str Name of the player output_sep : str (default: ' ') String for separating last name and first name in the output. firstname_output_letters : int Number of letters in the abbreviated first name. Returns gen_name : str The generalized name. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/text/generalize_names/","title":"generalize_names"},{"location":"api_modules/mlxtend.text/generalize_names_duplcheck/","text":"generalize_names_duplcheck generalize_names_duplcheck(df, col_name) Generalizes names and removes duplicates. Applies mlxtend.text.generalize_names to a DataFrame with 1 first name letter by default and uses more first name letters if duplicates are detected. Parameters df : pandas.DataFrame DataFrame that contains a column where generalize_names should be applied. col_name : str Name of the DataFrame column where generalize_names function should be applied to. Returns df_new : str New DataFrame object where generalize_names function has been applied without duplicates. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/text/generalize_names_duplcheck/","title":"Generalize names duplcheck"},{"location":"api_modules/mlxtend.text/generalize_names_duplcheck/#generalize_names_duplcheck","text":"generalize_names_duplcheck(df, col_name) Generalizes names and removes duplicates. Applies mlxtend.text.generalize_names to a DataFrame with 1 first name letter by default and uses more first name letters if duplicates are detected. Parameters df : pandas.DataFrame DataFrame that contains a column where generalize_names should be applied. col_name : str Name of the DataFrame column where generalize_names function should be applied to. Returns df_new : str New DataFrame object where generalize_names function has been applied without duplicates. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/text/generalize_names_duplcheck/","title":"generalize_names_duplcheck"},{"location":"api_modules/mlxtend.text/tokenizer_emoticons/","text":"tokenizer_emoticons tokenizer_emoticons(text) Return emoticons from text Examples >>> tokenizer_emoticons('</a>This :) is :( a test :-)!') [':)', ':(', ':-)'] For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/text/tokenizer_emoticons/","title":"Tokenizer emoticons"},{"location":"api_modules/mlxtend.text/tokenizer_emoticons/#tokenizer_emoticons","text":"tokenizer_emoticons(text) Return emoticons from text Examples >>> tokenizer_emoticons('</a>This :) is :( a test :-)!') [':)', ':(', ':-)'] For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/text/tokenizer_emoticons/","title":"tokenizer_emoticons"},{"location":"api_modules/mlxtend.text/tokenizer_words_and_emoticons/","text":"tokenizer_words_and_emoticons tokenizer_words_and_emoticons(text) Convert text to lowercase words and emoticons. Examples >>> tokenizer_words_and_emoticons('</a>This :) is :( a test :-)!') ['this', 'is', 'a', 'test', ':)', ':(', ':-)'] For more usage examples, please see https://rasbt.github.io/mlxtend/user_guide/text/tokenizer_words_and_emoticons/","title":"Tokenizer words and emoticons"},{"location":"api_modules/mlxtend.text/tokenizer_words_and_emoticons/#tokenizer_words_and_emoticons","text":"tokenizer_words_and_emoticons(text) Convert text to lowercase words and emoticons. Examples >>> tokenizer_words_and_emoticons('</a>This :) is :( a test :-)!') ['this', 'is', 'a', 'test', ':)', ':(', ':-)'] For more usage examples, please see https://rasbt.github.io/mlxtend/user_guide/text/tokenizer_words_and_emoticons/","title":"tokenizer_words_and_emoticons"},{"location":"api_modules/mlxtend.utils/Counter/","text":"Counter Counter(stderr=False, start_newline=True, precision=0, name=None) Class to display the progress of for-loop iterators. Parameters stderr : bool (default: True) Prints output to sys.stderr if True; uses sys.stdout otherwise. start_newline : bool (default: True) Prepends a new line to the counter, which prevents overwriting counters if multiple counters are printed in succession. precision: int (default: 0) Sets the number of decimal places when displaying the time elapsed in seconds. name : string (default: None) Prepends the specified name before the counter to allow distinguishing between multiple counters. Attributes curr_iter : int The current iteration. start_time : float The system's time in seconds when the Counter was initialized. end_time : float The system's time in seconds when the Counter was last updated. Examples >>> cnt = Counter() >>> for i in range(20): ... # do some computation ... time.sleep(0.1) ... cnt.update() 20 iter | 2 sec >>> print('The counter was initialized.' ' %d seconds ago.' % (time.time() - cnt.start_time)) The counter was initialized 2 seconds ago >>> print('The counter was last updated' ' %d seconds ago.' % (time.time() - cnt.end_time)) The counter was last updated 0 seconds ago. For more usage examples, please see https://rasbt.github.io/mlxtend/user_guide/utils/Counter/ Methods update() Print current iteration and time elapsed.","title":"Counter"},{"location":"api_modules/mlxtend.utils/Counter/#counter","text":"Counter(stderr=False, start_newline=True, precision=0, name=None) Class to display the progress of for-loop iterators. Parameters stderr : bool (default: True) Prints output to sys.stderr if True; uses sys.stdout otherwise. start_newline : bool (default: True) Prepends a new line to the counter, which prevents overwriting counters if multiple counters are printed in succession. precision: int (default: 0) Sets the number of decimal places when displaying the time elapsed in seconds. name : string (default: None) Prepends the specified name before the counter to allow distinguishing between multiple counters. Attributes curr_iter : int The current iteration. start_time : float The system's time in seconds when the Counter was initialized. end_time : float The system's time in seconds when the Counter was last updated. Examples >>> cnt = Counter() >>> for i in range(20): ... # do some computation ... time.sleep(0.1) ... cnt.update() 20 iter | 2 sec >>> print('The counter was initialized.' ' %d seconds ago.' % (time.time() - cnt.start_time)) The counter was initialized 2 seconds ago >>> print('The counter was last updated' ' %d seconds ago.' % (time.time() - cnt.end_time)) The counter was last updated 0 seconds ago. For more usage examples, please see https://rasbt.github.io/mlxtend/user_guide/utils/Counter/","title":"Counter"},{"location":"api_modules/mlxtend.utils/Counter/#methods","text":"update() Print current iteration and time elapsed.","title":"Methods"},{"location":"api_modules/mlxtend.utils/assert_raises/","text":"assert_raises assert_raises(exception_type, message, func, args, * kwargs) Check that an exception is raised with a specific message Parameters exception_type : exception The exception that should be raised message : str (default: None) The error message that should be raised. Ignored if False or None. func : callable The function that raises the exception *args : positional arguments to func . **kwargs : keyword arguments to func","title":"Assert raises"},{"location":"api_modules/mlxtend.utils/assert_raises/#assert_raises","text":"assert_raises(exception_type, message, func, args, * kwargs) Check that an exception is raised with a specific message Parameters exception_type : exception The exception that should be raised message : str (default: None) The error message that should be raised. Ignored if False or None. func : callable The function that raises the exception *args : positional arguments to func . **kwargs : keyword arguments to func","title":"assert_raises"},{"location":"api_modules/mlxtend.utils/check_Xy/","text":"check_Xy check_Xy(X, y, y_int=True) None","title":"check Xy"},{"location":"api_modules/mlxtend.utils/check_Xy/#check_xy","text":"check_Xy(X, y, y_int=True) None","title":"check_Xy"},{"location":"api_modules/mlxtend.utils/format_kwarg_dictionaries/","text":"format_kwarg_dictionaries format_kwarg_dictionaries(default_kwargs=None, user_kwargs=None, protected_keys=None) Function to combine default and user specified kwargs dictionaries Parameters default_kwargs : dict, optional Default kwargs (default is None). user_kwargs : dict, optional User specified kwargs (default is None). protected_keys : array_like, optional Sequence of keys to be removed from the returned dictionary (default is None). Returns formatted_kwargs : dict Formatted kwargs dictionary.","title":"Format kwarg dictionaries"},{"location":"api_modules/mlxtend.utils/format_kwarg_dictionaries/#format_kwarg_dictionaries","text":"format_kwarg_dictionaries(default_kwargs=None, user_kwargs=None, protected_keys=None) Function to combine default and user specified kwargs dictionaries Parameters default_kwargs : dict, optional Default kwargs (default is None). user_kwargs : dict, optional User specified kwargs (default is None). protected_keys : array_like, optional Sequence of keys to be removed from the returned dictionary (default is None). Returns formatted_kwargs : dict Formatted kwargs dictionary.","title":"format_kwarg_dictionaries"},{"location":"api_subpackages/mlxtend._base/","text":"mlxtend version: 0.23.1","title":"Mlxtend. base"},{"location":"api_subpackages/mlxtend.classifier/","text":"mlxtend version: 0.23.1 Adaline Adaline(eta=0.01, epochs=50, minibatches=None, random_seed=None, print_progress=0) ADAptive LInear NEuron classifier. Note that this implementation of Adaline expects binary class labels in {0, 1}. Parameters eta : float (default: 0.01) solver rate (between 0.0 and 1.0) epochs : int (default: 50) Passes over the training dataset. Prior to each epoch, the dataset is shuffled if minibatches > 1 to prevent cycles in stochastic gradient descent. minibatches : int (default: None) The number of minibatches for gradient-based optimization. If None: Normal Equations (closed-form solution) If 1: Gradient Descent learning If len(y): Stochastic Gradient Descent (SGD) online learning If 1 < minibatches < len(y): SGD Minibatch learning random_seed : int (default: None) Set random state for shuffling and initializing the weights. print_progress : int (default: 0) Prints progress in fitting to stderr if not solver='normal equation' 0: No output 1: Epochs elapsed and cost 2: 1 plus time elapsed 3: 2 plus estimated time until completion Attributes w_ : 2d-array, shape={n_features, 1} Model weights after fitting. b_ : 1d-array, shape={1,} Bias unit after fitting. cost_ : list Sum of squared errors after each epoch. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/classifier/Adaline/ Methods fit(X, y, init_params=True) Learn model from training data. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values. init_params : bool (default: True) Re-initializes model parameters prior to fitting. Set False to continue training with weights from a previous model fitting. Returns self : object get_params(deep=True) Get parameters for this estimator. Parameters deep : boolean, optional If True, will return the parameters for this estimator and contained subobjects that are estimators. Returns params : mapping of string to any Parameter names mapped to their values.' adapted from https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/base.py Author: Gael Varoquaux gael.varoquaux@normalesup.org License: BSD 3 clause predict(X) Predict targets from X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns target_values : array-like, shape = [n_samples] Predicted target values. score(X, y) Compute the prediction accuracy Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values (true class labels). Returns acc : float The prediction accuracy as a float between 0.0 and 1.0 (perfect score). set_params( params) Set the parameters of this estimator. The method works on simple estimators as well as on nested objects (such as pipelines). The latter have parameters of the form <component>__<parameter> so that it's possible to update each component of a nested object. Returns self adapted from https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/base.py Author: Gael Varoquaux <gael.varoquaux@normalesup.org> License: BSD 3 clause EnsembleVoteClassifier EnsembleVoteClassifier(clfs, voting='hard', weights=None, verbose=0, use_clones=True, fit_base_estimators=True) Soft Voting/Majority Rule classifier for scikit-learn estimators. Parameters clfs : array-like, shape = [n_classifiers] A list of classifiers. Invoking the fit method on the VotingClassifier will fit clones of those original classifiers be stored in the class attribute if use_clones=True (default) and fit_base_estimators=True (default). voting : str, {'hard', 'soft'} (default='hard') If 'hard', uses predicted class labels for majority rule voting. Else if 'soft', predicts the class label based on the argmax of the sums of the predicted probalities, which is recommended for an ensemble of well-calibrated classifiers. weights : array-like, shape = [n_classifiers], optional (default= None ) Sequence of weights ( float or int ) to weight the occurances of predicted class labels ( hard voting) or class probabilities before averaging ( soft voting). Uses uniform weights if None . verbose : int, optional (default=0) Controls the verbosity of the building process. - verbose=0 (default): Prints nothing - verbose=1 : Prints the number & name of the clf being fitted - verbose=2 : Prints info about the parameters of the clf being fitted - verbose>2 : Changes verbose param of the underlying clf to self.verbose - 2 use_clones : bool (default: True) Clones the classifiers for stacking classification if True (default) or else uses the original ones, which will be refitted on the dataset upon calling the fit method. Hence, if use_clones=True, the original input classifiers will remain unmodified upon using the StackingClassifier's fit method. Setting use_clones=False is recommended if you are working with estimators that are supporting the scikit-learn fit/predict API interface but are not compatible to scikit-learn's clone function. fit_base_estimators : bool (default: True) Refits classifiers in clfs if True; uses references to the clfs , otherwise (assumes that the classifiers were already fit). Note: fit_base_estimators=False will enforce use_clones to be False, and is incompatible to most scikit-learn wrappers! For instance, if any form of cross-validation is performed this would require the re-fitting classifiers to training folds, which would raise a NotFitterError if fit_base_estimators=False. (New in mlxtend v0.6.) Attributes classes_ : array-like, shape = [n_predictions] clf : array-like, shape = [n_predictions] The input classifiers; may be overwritten if use_clones=False clf_ : array-like, shape = [n_predictions] Fitted input classifiers; clones if use_clones=True Examples ``` >>> import numpy as np >>> from sklearn.linear_model import LogisticRegression >>> from sklearn.naive_bayes import GaussianNB >>> from sklearn.ensemble import RandomForestClassifier >>> from mlxtend.sklearn import EnsembleVoteClassifier >>> clf1 = LogisticRegression(random_seed=1) >>> clf2 = RandomForestClassifier(random_seed=1) >>> clf3 = GaussianNB() >>> X = np.array([[-1, -1], [-2, -1], [-3, -2], [1, 1], [2, 1], [3, 2]]) >>> y = np.array([1, 1, 1, 2, 2, 2]) >>> eclf1 = EnsembleVoteClassifier(clfs=[clf1, clf2, clf3], ... voting='hard', verbose=1) >>> eclf1 = eclf1.fit(X, y) >>> print(eclf1.predict(X)) [1 1 1 2 2 2] >>> eclf2 = EnsembleVoteClassifier(clfs=[clf1, clf2, clf3], voting='soft') >>> eclf2 = eclf2.fit(X, y) >>> print(eclf2.predict(X)) [1 1 1 2 2 2] >>> eclf3 = EnsembleVoteClassifier(clfs=[clf1, clf2, clf3], ... voting='soft', weights=[2,1,1]) >>> eclf3 = eclf3.fit(X, y) >>> print(eclf3.predict(X)) [1 1 1 2 2 2] >>> For more usage examples, please see https://rasbt.github.io/mlxtend/user_guide/classifier/EnsembleVoteClassifier/ ``` Methods fit(X, y, sample_weight=None) Learn weight coefficients from training data for each classifier. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values. sample_weight : array-like, shape = [n_samples], optional Sample weights passed as sample_weights to each regressor in the regressors list as well as the meta_regressor. Raises error if some regressor does not support sample_weight in the fit() method. Returns self : object fit_transform(X, y=None, fit_params) Fit to data, then transform it. Fits transformer to `X` and `y` with optional parameters `fit_params` and returns a transformed version of `X`. Parameters X : array-like of shape (n_samples, n_features) Input samples. y : array-like of shape (n_samples,) or (n_samples, n_outputs), default=None Target values (None for unsupervised transformations). **fit_params : dict Additional fit parameters. Returns X_new : ndarray array of shape (n_samples, n_features_new) Transformed array. get_params(deep=True) Return estimator parameter names for GridSearch support. predict(X) Predict class labels for X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns maj : array-like, shape = [n_samples] Predicted class labels. predict_proba(X) Predict class probabilities for X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns avg : array-like, shape = [n_samples, n_classes] Weighted average probability for each class per sample. score(X, y, sample_weight=None) Return the mean accuracy on the given test data and labels. In multi-label classification, this is the subset accuracy which is a harsh metric since you require for each sample that each label set be correctly predicted. Parameters X : array-like of shape (n_samples, n_features) Test samples. y : array-like of shape (n_samples,) or (n_samples, n_outputs) True labels for X . sample_weight : array-like of shape (n_samples,), default=None Sample weights. Returns score : float Mean accuracy of self.predict(X) w.r.t. y . set_output( , transform=None)* Set output container. See :ref:`sphx_glr_auto_examples_miscellaneous_plot_set_output.py` for an example on how to use the API. Parameters transform : {\"default\", \"pandas\"}, default=None Configure output of transform and fit_transform . \"default\" : Default output format of a transformer \"pandas\" : DataFrame output None : Transform configuration is unchanged Returns self : estimator instance Estimator instance. set_params( params) Set the parameters of this estimator. The method works on simple estimators as well as on nested objects (such as :class:`~sklearn.pipeline.Pipeline`). The latter have parameters of the form ``<component>__<parameter>`` so that it's possible to update each component of a nested object. Parameters **params : dict Estimator parameters. Returns self : estimator instance Estimator instance. transform(X) Return class labels or probabilities for X for each estimator. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns If voting='soft'`` : array-like = [n_classifiers, n_samples, n_classes] Class probabilties calculated by each classifier. If voting='hard'`` : array-like = [n_classifiers, n_samples] Class labels predicted by each classifier. LogisticRegression LogisticRegression(eta=0.01, epochs=50, l2_lambda=0.0, minibatches=1, random_seed=None, print_progress=0) Logistic regression classifier. Note that this implementation of Logistic Regression expects binary class labels in {0, 1}. Parameters eta : float (default: 0.01) Learning rate (between 0.0 and 1.0) epochs : int (default: 50) Passes over the training dataset. Prior to each epoch, the dataset is shuffled if minibatches > 1 to prevent cycles in stochastic gradient descent. l2_lambda : float Regularization parameter for L2 regularization. No regularization if l2_lambda=0.0. minibatches : int (default: 1) The number of minibatches for gradient-based optimization. If 1: Gradient Descent learning If len(y): Stochastic Gradient Descent (SGD) online learning If 1 < minibatches < len(y): SGD Minibatch learning random_seed : int (default: None) Set random state for shuffling and initializing the weights. print_progress : int (default: 0) Prints progress in fitting to stderr. 0: No output 1: Epochs elapsed and cost 2: 1 plus time elapsed 3: 2 plus estimated time until completion Attributes w_ : 2d-array, shape={n_features, 1} Model weights after fitting. b_ : 1d-array, shape={1,} Bias unit after fitting. cost_ : list List of floats with cross_entropy cost (sgd or gd) for every epoch. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/classifier/LogisticRegression/ Methods fit(X, y, init_params=True) Learn model from training data. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values. init_params : bool (default: True) Re-initializes model parameters prior to fitting. Set False to continue training with weights from a previous model fitting. Returns self : object get_params(deep=True) Get parameters for this estimator. Parameters deep : boolean, optional If True, will return the parameters for this estimator and contained subobjects that are estimators. Returns params : mapping of string to any Parameter names mapped to their values.' adapted from https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/base.py Author: Gael Varoquaux gael.varoquaux@normalesup.org License: BSD 3 clause predict(X) Predict targets from X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns target_values : array-like, shape = [n_samples] Predicted target values. predict_proba(X) Predict class probabilities of X from the net input. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns Class 1 probability : float score(X, y) Compute the prediction accuracy Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values (true class labels). Returns acc : float The prediction accuracy as a float between 0.0 and 1.0 (perfect score). set_params( params) Set the parameters of this estimator. The method works on simple estimators as well as on nested objects (such as pipelines). The latter have parameters of the form <component>__<parameter> so that it's possible to update each component of a nested object. Returns self adapted from https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/base.py Author: Gael Varoquaux <gael.varoquaux@normalesup.org> License: BSD 3 clause MultiLayerPerceptron MultiLayerPerceptron(eta=0.5, epochs=50, hidden_layers=[50], n_classes=None, momentum=0.0, l1=0.0, l2=0.0, dropout=1.0, decrease_const=0.0, minibatches=1, random_seed=None, print_progress=0) Multi-layer perceptron classifier with logistic sigmoid activations Parameters eta : float (default: 0.5) Learning rate (between 0.0 and 1.0) epochs : int (default: 50) Passes over the training dataset. Prior to each epoch, the dataset is shuffled if minibatches > 1 to prevent cycles in stochastic gradient descent. hidden_layers : list (default: [50]) Number of units per hidden layer. By default 50 units in the first hidden layer. At the moment only 1 hidden layer is supported n_classes : int (default: None) A positive integer to declare the number of class labels if not all class labels are present in a partial training set. Gets the number of class labels automatically if None. l1 : float (default: 0.0) L1 regularization strength l2 : float (default: 0.0) L2 regularization strength momentum : float (default: 0.0) Momentum constant. Factor multiplied with the gradient of the previous epoch t-1 to improve learning speed w(t) := w(t) - (grad(t) + momentum * grad(t-1)) decrease_const : float (default: 0.0) Decrease constant. Shrinks the learning rate after each epoch via eta / (1 + epoch*decrease_const) minibatches : int (default: 1) Divide the training data into k minibatches for accelerated stochastic gradient descent learning. Gradient Descent Learning if minibatches = 1 Stochastic Gradient Descent learning if minibatches = len(y) Minibatch learning if minibatches > 1 random_seed : int (default: None) Set random state for shuffling and initializing the weights. print_progress : int (default: 0) Prints progress in fitting to stderr. 0: No output 1: Epochs elapsed and cost 2: 1 plus time elapsed 3: 2 plus estimated time until completion Attributes w_ : 2d-array, shape=[n_features, n_classes] Weights after fitting. b_ : 1D-array, shape=[n_classes] Bias units after fitting. cost_ : list List of floats; the mean categorical cross entropy cost after each epoch. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/classifier/MultiLayerPerceptron/ Methods fit(X, y, init_params=True) Learn model from training data. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values. init_params : bool (default: True) Re-initializes model parameters prior to fitting. Set False to continue training with weights from a previous model fitting. Returns self : object get_params(deep=True) Get parameters for this estimator. Parameters deep : boolean, optional If True, will return the parameters for this estimator and contained subobjects that are estimators. Returns params : mapping of string to any Parameter names mapped to their values.' adapted from https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/base.py Author: Gael Varoquaux gael.varoquaux@normalesup.org License: BSD 3 clause predict(X) Predict targets from X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns target_values : array-like, shape = [n_samples] Predicted target values. predict_proba(X) Predict class probabilities of X from the net input. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns Class probabilties : array-like, shape= [n_samples, n_classes] score(X, y) Compute the prediction accuracy Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values (true class labels). Returns acc : float The prediction accuracy as a float between 0.0 and 1.0 (perfect score). set_params( params) Set the parameters of this estimator. The method works on simple estimators as well as on nested objects (such as pipelines). The latter have parameters of the form <component>__<parameter> so that it's possible to update each component of a nested object. Returns self adapted from https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/base.py Author: Gael Varoquaux <gael.varoquaux@normalesup.org> License: BSD 3 clause OneRClassifier OneRClassifier(resolve_ties='first') OneR (One Rule) Classifier. Parameters resolve_ties : str (default: 'first') Option for how to resolve ties if two or more features have the same error. Options are - 'first' (default): chooses first feature in the list, i.e., feature with the lower column index. - 'chi-squared': performs a chi-squared test for each feature against the target and selects the feature with the lowest p-value. Attributes self.classes_labels_ : array-like, shape = [n_labels] Array containing the unique class labels found in the training set. self.feature_idx_ : int The index of the rules' feature based on the column in the training set. self.p_value_ : float The p value for a given feature. Only available after calling fit when the OneR attribute resolve_ties = 'chi-squared' is set. self.prediction_dict_ : dict Dictionary containing information about the feature's (self.feature_idx_) rules and total error. E.g., {'total error': 37, 'rules (value: class)': {0: 0, 1: 2}} means the total error is 37, and the rules are \"if feature value == 0 classify as 0\" and \"if feature value == 1 classify as 2\". (And classify as class 1 otherwise.) For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/classifier/OneRClassifier/ Methods fit(X, y) Learn rule from training data. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values. Returns self : object get_params(deep=True) Get parameters for this estimator. Parameters deep : bool, default=True If True, will return the parameters for this estimator and contained subobjects that are estimators. Returns params : dict Parameter names mapped to their values. predict(X) Predict class labels for X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns maj : array-like, shape = [n_samples] Predicted class labels. score(X, y, sample_weight=None) Return the mean accuracy on the given test data and labels. In multi-label classification, this is the subset accuracy which is a harsh metric since you require for each sample that each label set be correctly predicted. Parameters X : array-like of shape (n_samples, n_features) Test samples. y : array-like of shape (n_samples,) or (n_samples, n_outputs) True labels for X . sample_weight : array-like of shape (n_samples,), default=None Sample weights. Returns score : float Mean accuracy of self.predict(X) w.r.t. y . set_params( params) Set the parameters of this estimator. The method works on simple estimators as well as on nested objects (such as :class:`~sklearn.pipeline.Pipeline`). The latter have parameters of the form ``<component>__<parameter>`` so that it's possible to update each component of a nested object. Parameters **params : dict Estimator parameters. Returns self : estimator instance Estimator instance. Perceptron Perceptron(eta=0.1, epochs=50, random_seed=None, print_progress=0) Perceptron classifier. Note that this implementation of the Perceptron expects binary class labels in {0, 1}. Parameters eta : float (default: 0.1) Learning rate (between 0.0 and 1.0) epochs : int (default: 50) Number of passes over the training dataset. Prior to each epoch, the dataset is shuffled to prevent cycles. random_seed : int Random state for initializing random weights and shuffling. print_progress : int (default: 0) Prints progress in fitting to stderr. 0: No output 1: Epochs elapsed and cost 2: 1 plus time elapsed 3: 2 plus estimated time until completion Attributes w_ : 2d-array, shape={n_features, 1} Model weights after fitting. b_ : 1d-array, shape={1,} Bias unit after fitting. cost_ : list Number of misclassifications in every epoch. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/classifier/Perceptron/ Methods fit(X, y, init_params=True) Learn model from training data. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values. init_params : bool (default: True) Re-initializes model parameters prior to fitting. Set False to continue training with weights from a previous model fitting. Returns self : object get_params(deep=True) Get parameters for this estimator. Parameters deep : boolean, optional If True, will return the parameters for this estimator and contained subobjects that are estimators. Returns params : mapping of string to any Parameter names mapped to their values.' adapted from https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/base.py Author: Gael Varoquaux gael.varoquaux@normalesup.org License: BSD 3 clause predict(X) Predict targets from X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns target_values : array-like, shape = [n_samples] Predicted target values. score(X, y) Compute the prediction accuracy Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values (true class labels). Returns acc : float The prediction accuracy as a float between 0.0 and 1.0 (perfect score). set_params( params) Set the parameters of this estimator. The method works on simple estimators as well as on nested objects (such as pipelines). The latter have parameters of the form <component>__<parameter> so that it's possible to update each component of a nested object. Returns self adapted from https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/base.py Author: Gael Varoquaux <gael.varoquaux@normalesup.org> License: BSD 3 clause SoftmaxRegression SoftmaxRegression(eta=0.01, epochs=50, l2=0.0, minibatches=1, n_classes=None, random_seed=None, print_progress=0) Softmax regression classifier. Parameters eta : float (default: 0.01) Learning rate (between 0.0 and 1.0) epochs : int (default: 50) Passes over the training dataset. Prior to each epoch, the dataset is shuffled if minibatches > 1 to prevent cycles in stochastic gradient descent. l2 : float Regularization parameter for L2 regularization. No regularization if l2=0.0. minibatches : int (default: 1) The number of minibatches for gradient-based optimization. If 1: Gradient Descent learning If len(y): Stochastic Gradient Descent (SGD) online learning If 1 < minibatches < len(y): SGD Minibatch learning n_classes : int (default: None) A positive integer to declare the number of class labels if not all class labels are present in a partial training set. Gets the number of class labels automatically if None. random_seed : int (default: None) Set random state for shuffling and initializing the weights. print_progress : int (default: 0) Prints progress in fitting to stderr. 0: No output 1: Epochs elapsed and cost 2: 1 plus time elapsed 3: 2 plus estimated time until completion Attributes w_ : 2d-array, shape={n_features, 1} Model weights after fitting. b_ : 1d-array, shape={1,} Bias unit after fitting. cost_ : list List of floats, the average cross_entropy for each epoch. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/classifier/SoftmaxRegression/ Methods fit(X, y, init_params=True) Learn model from training data. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values. init_params : bool (default: True) Re-initializes model parameters prior to fitting. Set False to continue training with weights from a previous model fitting. Returns self : object get_params(deep=True) Get parameters for this estimator. Parameters deep : boolean, optional If True, will return the parameters for this estimator and contained subobjects that are estimators. Returns params : mapping of string to any Parameter names mapped to their values.' adapted from https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/base.py Author: Gael Varoquaux gael.varoquaux@normalesup.org License: BSD 3 clause predict(X) Predict targets from X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns target_values : array-like, shape = [n_samples] Predicted target values. predict_proba(X) Predict class probabilities of X from the net input. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns Class probabilties : array-like, shape= [n_samples, n_classes] score(X, y) Compute the prediction accuracy Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values (true class labels). Returns acc : float The prediction accuracy as a float between 0.0 and 1.0 (perfect score). set_params( params) Set the parameters of this estimator. The method works on simple estimators as well as on nested objects (such as pipelines). The latter have parameters of the form <component>__<parameter> so that it's possible to update each component of a nested object. Returns self adapted from https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/base.py Author: Gael Varoquaux <gael.varoquaux@normalesup.org> License: BSD 3 clause StackingCVClassifier StackingCVClassifier(classifiers, meta_classifier, use_probas=False, drop_proba_col=None, cv=2, shuffle=True, random_state=None, stratify=True, verbose=0, use_features_in_secondary=False, store_train_meta_features=False, use_clones=True, n_jobs=None, pre_dispatch='2 n_jobs')* A 'Stacking Cross-Validation' classifier for scikit-learn estimators. New in mlxtend v0.4.3 Parameters classifiers : array-like, shape = [n_classifiers] A list of classifiers. Invoking the fit method on the StackingCVClassifer will fit clones of these original classifiers that will be stored in the class attribute self.clfs_ if use_clones=True . meta_classifier : object The meta-classifier to be fitted on the ensemble of classifiers use_probas : bool (default: False) If True, trains meta-classifier based on predicted probabilities instead of class labels. drop_proba_col : string (default: None) Drops extra \"probability\" column in the feature set, because it is redundant: p(y_c) = 1 - p(y_1) + p(y_2) + ... + p(y_{c-1}). This can be useful for meta-classifiers that are sensitive to perfectly collinear features. If 'last', drops last probability column. If 'first', drops first probability column. Only relevant if use_probas=True . cv : int, cross-validation generator or an iterable, optional (default: 2) Determines the cross-validation splitting strategy. Possible inputs for cv are: - None, to use the default 2-fold cross validation, - integer, to specify the number of folds in a (Stratified)KFold , - An object to be used as a cross-validation generator. - An iterable yielding train, test splits. For integer/None inputs, it will use either a KFold or StratifiedKFold cross validation depending the value of stratify argument. shuffle : bool (default: True) If True, and the cv argument is integer, the training data will be shuffled at fitting stage prior to cross-validation. If the cv argument is a specific cross validation technique, this argument is omitted. random_state : int, RandomState instance or None, optional (default: None) Constrols the randomness of the cv splitter. Used when cv is integer and shuffle=True . New in v0.16.0. stratify : bool (default: True) If True, and the cv argument is integer it will follow a stratified K-Fold cross validation technique. If the cv argument is a specific cross validation technique, this argument is omitted. verbose : int, optional (default=0) Controls the verbosity of the building process. - verbose=0 (default): Prints nothing - verbose=1 : Prints the number & name of the regressor being fitted and which fold is currently being used for fitting - verbose=2 : Prints info about the parameters of the regressor being fitted - verbose>2 : Changes verbose param of the underlying regressor to self.verbose - 2 use_features_in_secondary : bool (default: False) If True, the meta-classifier will be trained both on the predictions of the original classifiers and the original dataset. If False, the meta-classifier will be trained only on the predictions of the original classifiers. store_train_meta_features : bool (default: False) If True, the meta-features computed from the training data used for fitting the meta-classifier stored in the self.train_meta_features_ array, which can be accessed after calling fit . use_clones : bool (default: True) Clones the classifiers for stacking classification if True (default) or else uses the original ones, which will be refitted on the dataset upon calling the fit method. Hence, if use_clones=True, the original input classifiers will remain unmodified upon using the StackingCVClassifier's fit method. Setting use_clones=False is recommended if you are working with estimators that are supporting the scikit-learn fit/predict API interface but are not compatible to scikit-learn's clone function. n_jobs : int or None, optional (default=None) The number of CPUs to use to do the computation. None means 1 unless in a joblib.parallel_backend context. -1 means using all processors. for more details. New in v0.16.0. pre_dispatch : int, or string, optional Controls the number of jobs that get dispatched during parallel execution. Reducing this number can be useful to avoid an explosion of memory consumption when more jobs get dispatched than CPUs can process. This parameter can be: - None, in which case all the jobs are immediately created and spawned. Use this for lightweight and fast-running jobs, to avoid delays due to on-demand spawning of the jobs - An int, giving the exact number of total jobs that are spawned - A string, giving an expression as a function of n_jobs, as in '2*n_jobs' New in v0.16.0. Attributes clfs_ : list, shape=[n_classifiers] Fitted classifiers (clones of the original classifiers) meta_clf_ : estimator Fitted meta-classifier (clone of the original meta-estimator) train_meta_features : numpy array, shape = [n_samples, n_classifiers] meta-features for training data, where n_samples is the number of samples in training data and n_classifiers is the number of classfiers. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/classifier/StackingCVClassifier/ Methods decision_function(X) Predict class confidence scores for X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns scores : shape=(n_samples,) if n_classes == 2 else (n_samples, n_classes). Confidence scores per (sample, class) combination. In the binary case, confidence score for self.classes_[1] where >0 means this class would be predicted. fit(X, y, groups=None, sample_weight=None) Fit ensemble classifers and the meta-classifier. Parameters X : numpy array, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : numpy array, shape = [n_samples] Target values. groups : numpy array/None, shape = [n_samples] The group that each sample belongs to. This is used by specific folding strategies such as GroupKFold() sample_weight : array-like, shape = [n_samples], optional Sample weights passed as sample_weights to each regressor in the regressors list as well as the meta_regressor. Raises error if some regressor does not support sample_weight in the fit() method. Returns self : object fit_transform(X, y=None, fit_params) Fit to data, then transform it. Fits transformer to `X` and `y` with optional parameters `fit_params` and returns a transformed version of `X`. Parameters X : array-like of shape (n_samples, n_features) Input samples. y : array-like of shape (n_samples,) or (n_samples, n_outputs), default=None Target values (None for unsupervised transformations). **fit_params : dict Additional fit parameters. Returns X_new : ndarray array of shape (n_samples, n_features_new) Transformed array. get_params(deep=True) Return estimator parameter names for GridSearch support. predict(X) Predict target values for X. Parameters X : numpy array, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns labels : array-like, shape = [n_samples] Predicted class labels. predict_meta_features(X) Get meta-features of test-data. Parameters X : numpy array, shape = [n_samples, n_features] Test vectors, where n_samples is the number of samples and n_features is the number of features. Returns meta-features : numpy array, shape = [n_samples, n_classifiers] Returns the meta-features for test data. predict_proba(X) Predict class probabilities for X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns proba : array-like, shape = [n_samples, n_classes] or a list of n_outputs of such arrays if n_outputs > 1. Probability for each class per sample. score(X, y, sample_weight=None) Return the mean accuracy on the given test data and labels. In multi-label classification, this is the subset accuracy which is a harsh metric since you require for each sample that each label set be correctly predicted. Parameters X : array-like of shape (n_samples, n_features) Test samples. y : array-like of shape (n_samples,) or (n_samples, n_outputs) True labels for X . sample_weight : array-like of shape (n_samples,), default=None Sample weights. Returns score : float Mean accuracy of self.predict(X) w.r.t. y . set_output( , transform=None)* Set output container. See :ref:`sphx_glr_auto_examples_miscellaneous_plot_set_output.py` for an example on how to use the API. Parameters transform : {\"default\", \"pandas\"}, default=None Configure output of transform and fit_transform . \"default\" : Default output format of a transformer \"pandas\" : DataFrame output None : Transform configuration is unchanged Returns self : estimator instance Estimator instance. set_params( params) Set the parameters of this estimator. Valid parameter keys can be listed with ``get_params()``. Returns self Properties named_classifiers None StackingClassifier StackingClassifier(classifiers, meta_classifier, use_probas=False, drop_proba_col=None, average_probas=False, verbose=0, use_features_in_secondary=False, store_train_meta_features=False, use_clones=True, fit_base_estimators=True) A Stacking classifier for scikit-learn estimators for classification. Parameters classifiers : array-like, shape = [n_classifiers] A list of classifiers. Invoking the fit method on the StackingClassifer will fit clones of these original classifiers that will be stored in the class attribute self.clfs_ if use_clones=True (default) and fit_base_estimators=True (default). meta_classifier : object The meta-classifier to be fitted on the ensemble of classifiers use_probas : bool (default: False) If True, trains meta-classifier based on predicted probabilities instead of class labels. drop_proba_col : string (default: None) Drops extra \"probability\" column in the feature set, because it is redundant: p(y_c) = 1 - p(y_1) + p(y_2) + ... + p(y_{c-1}). This can be useful for meta-classifiers that are sensitive to perfectly collinear features. If 'last', drops last probability column. If 'first', drops first probability column. Only relevant if use_probas=True . average_probas : bool (default: False) Averages the probabilities as meta features if True . Only relevant if use_probas=True . verbose : int, optional (default=0) Controls the verbosity of the building process. - verbose=0 (default): Prints nothing - verbose=1 : Prints the number & name of the regressor being fitted - verbose=2 : Prints info about the parameters of the regressor being fitted - verbose>2 : Changes verbose param of the underlying regressor to self.verbose - 2 use_features_in_secondary : bool (default: False) If True, the meta-classifier will be trained both on the predictions of the original classifiers and the original dataset. If False, the meta-classifier will be trained only on the predictions of the original classifiers. store_train_meta_features : bool (default: False) If True, the meta-features computed from the training data used for fitting the meta-classifier stored in the self.train_meta_features_ array, which can be accessed after calling fit . use_clones : bool (default: True) Clones the classifiers for stacking classification if True (default) or else uses the original ones, which will be refitted on the dataset upon calling the fit method. Hence, if use_clones=True, the original input classifiers will remain unmodified upon using the StackingClassifier's fit method. Setting use_clones=False is recommended if you are working with estimators that are supporting the scikit-learn fit/predict API interface but are not compatible to scikit-learn's clone function. fit_base_estimators: bool (default: True) Refits classifiers in classifiers if True; uses references to the classifiers , otherwise (assumes that the classifiers were already fit). Note: fit_base_estimators=False will enforce use_clones to be False, and is incompatible to most scikit-learn wrappers! For instance, if any form of cross-validation is performed this would require the re-fitting classifiers to training folds, which would raise a NotFitterError if fit_base_estimators=False. (New in mlxtend v0.6.) Attributes clfs_ : list, shape=[n_classifiers] Fitted classifiers (clones of the original classifiers) meta_clf_ : estimator Fitted meta-classifier (clone of the original meta-estimator) train_meta_features : numpy array, shape = [n_samples, n_classifiers] meta-features for training data, where n_samples is the number of samples in training data and n_classifiers is the number of classfiers. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/classifier/StackingClassifier/ Methods decision_function(X) Predict class confidence scores for X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns scores : shape=(n_samples,) if n_classes == 2 else (n_samples, n_classes). Confidence scores per (sample, class) combination. In the binary case, confidence score for self.classes_[1] where >0 means this class would be predicted. fit(X, y, sample_weight=None) Fit ensemble classifers and the meta-classifier. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] or [n_samples, n_outputs] Target values. sample_weight : array-like, shape = [n_samples], optional Sample weights passed as sample_weights to each regressor in the regressors list as well as the meta_regressor. Raises error if some regressor does not support sample_weight in the fit() method. Returns self : object fit_transform(X, y=None, fit_params) Fit to data, then transform it. Fits transformer to `X` and `y` with optional parameters `fit_params` and returns a transformed version of `X`. Parameters X : array-like of shape (n_samples, n_features) Input samples. y : array-like of shape (n_samples,) or (n_samples, n_outputs), default=None Target values (None for unsupervised transformations). **fit_params : dict Additional fit parameters. Returns X_new : ndarray array of shape (n_samples, n_features_new) Transformed array. get_params(deep=True) Return estimator parameter names for GridSearch support. predict(X) Predict target values for X. Parameters X : numpy array, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns labels : array-like, shape = [n_samples] Predicted class labels. predict_meta_features(X) Get meta-features of test-data. Parameters X : numpy array, shape = [n_samples, n_features] Test vectors, where n_samples is the number of samples and n_features is the number of features. Returns meta-features : numpy array, shape = [n_samples, n_classifiers] Returns the meta-features for test data. predict_proba(X) Predict class probabilities for X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns proba : array-like, shape = [n_samples, n_classes] or a list of n_outputs of such arrays if n_outputs > 1. Probability for each class per sample. score(X, y, sample_weight=None) Return the mean accuracy on the given test data and labels. In multi-label classification, this is the subset accuracy which is a harsh metric since you require for each sample that each label set be correctly predicted. Parameters X : array-like of shape (n_samples, n_features) Test samples. y : array-like of shape (n_samples,) or (n_samples, n_outputs) True labels for X . sample_weight : array-like of shape (n_samples,), default=None Sample weights. Returns score : float Mean accuracy of self.predict(X) w.r.t. y . set_output( , transform=None)* Set output container. See :ref:`sphx_glr_auto_examples_miscellaneous_plot_set_output.py` for an example on how to use the API. Parameters transform : {\"default\", \"pandas\"}, default=None Configure output of transform and fit_transform . \"default\" : Default output format of a transformer \"pandas\" : DataFrame output None : Transform configuration is unchanged Returns self : estimator instance Estimator instance. set_params( params) Set the parameters of this estimator. Valid parameter keys can be listed with ``get_params()``. Returns self Properties named_classifiers None","title":"Mlxtend.classifier"},{"location":"api_subpackages/mlxtend.classifier/#adaline","text":"Adaline(eta=0.01, epochs=50, minibatches=None, random_seed=None, print_progress=0) ADAptive LInear NEuron classifier. Note that this implementation of Adaline expects binary class labels in {0, 1}. Parameters eta : float (default: 0.01) solver rate (between 0.0 and 1.0) epochs : int (default: 50) Passes over the training dataset. Prior to each epoch, the dataset is shuffled if minibatches > 1 to prevent cycles in stochastic gradient descent. minibatches : int (default: None) The number of minibatches for gradient-based optimization. If None: Normal Equations (closed-form solution) If 1: Gradient Descent learning If len(y): Stochastic Gradient Descent (SGD) online learning If 1 < minibatches < len(y): SGD Minibatch learning random_seed : int (default: None) Set random state for shuffling and initializing the weights. print_progress : int (default: 0) Prints progress in fitting to stderr if not solver='normal equation' 0: No output 1: Epochs elapsed and cost 2: 1 plus time elapsed 3: 2 plus estimated time until completion Attributes w_ : 2d-array, shape={n_features, 1} Model weights after fitting. b_ : 1d-array, shape={1,} Bias unit after fitting. cost_ : list Sum of squared errors after each epoch. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/classifier/Adaline/","title":"Adaline"},{"location":"api_subpackages/mlxtend.classifier/#methods","text":"fit(X, y, init_params=True) Learn model from training data. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values. init_params : bool (default: True) Re-initializes model parameters prior to fitting. Set False to continue training with weights from a previous model fitting. Returns self : object get_params(deep=True) Get parameters for this estimator. Parameters deep : boolean, optional If True, will return the parameters for this estimator and contained subobjects that are estimators. Returns params : mapping of string to any Parameter names mapped to their values.' adapted from https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/base.py Author: Gael Varoquaux gael.varoquaux@normalesup.org License: BSD 3 clause predict(X) Predict targets from X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns target_values : array-like, shape = [n_samples] Predicted target values. score(X, y) Compute the prediction accuracy Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values (true class labels). Returns acc : float The prediction accuracy as a float between 0.0 and 1.0 (perfect score). set_params( params) Set the parameters of this estimator. The method works on simple estimators as well as on nested objects (such as pipelines). The latter have parameters of the form <component>__<parameter> so that it's possible to update each component of a nested object. Returns self adapted from https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/base.py Author: Gael Varoquaux <gael.varoquaux@normalesup.org> License: BSD 3 clause","title":"Methods"},{"location":"api_subpackages/mlxtend.classifier/#ensemblevoteclassifier","text":"EnsembleVoteClassifier(clfs, voting='hard', weights=None, verbose=0, use_clones=True, fit_base_estimators=True) Soft Voting/Majority Rule classifier for scikit-learn estimators. Parameters clfs : array-like, shape = [n_classifiers] A list of classifiers. Invoking the fit method on the VotingClassifier will fit clones of those original classifiers be stored in the class attribute if use_clones=True (default) and fit_base_estimators=True (default). voting : str, {'hard', 'soft'} (default='hard') If 'hard', uses predicted class labels for majority rule voting. Else if 'soft', predicts the class label based on the argmax of the sums of the predicted probalities, which is recommended for an ensemble of well-calibrated classifiers. weights : array-like, shape = [n_classifiers], optional (default= None ) Sequence of weights ( float or int ) to weight the occurances of predicted class labels ( hard voting) or class probabilities before averaging ( soft voting). Uses uniform weights if None . verbose : int, optional (default=0) Controls the verbosity of the building process. - verbose=0 (default): Prints nothing - verbose=1 : Prints the number & name of the clf being fitted - verbose=2 : Prints info about the parameters of the clf being fitted - verbose>2 : Changes verbose param of the underlying clf to self.verbose - 2 use_clones : bool (default: True) Clones the classifiers for stacking classification if True (default) or else uses the original ones, which will be refitted on the dataset upon calling the fit method. Hence, if use_clones=True, the original input classifiers will remain unmodified upon using the StackingClassifier's fit method. Setting use_clones=False is recommended if you are working with estimators that are supporting the scikit-learn fit/predict API interface but are not compatible to scikit-learn's clone function. fit_base_estimators : bool (default: True) Refits classifiers in clfs if True; uses references to the clfs , otherwise (assumes that the classifiers were already fit). Note: fit_base_estimators=False will enforce use_clones to be False, and is incompatible to most scikit-learn wrappers! For instance, if any form of cross-validation is performed this would require the re-fitting classifiers to training folds, which would raise a NotFitterError if fit_base_estimators=False. (New in mlxtend v0.6.) Attributes classes_ : array-like, shape = [n_predictions] clf : array-like, shape = [n_predictions] The input classifiers; may be overwritten if use_clones=False clf_ : array-like, shape = [n_predictions] Fitted input classifiers; clones if use_clones=True Examples ``` >>> import numpy as np >>> from sklearn.linear_model import LogisticRegression >>> from sklearn.naive_bayes import GaussianNB >>> from sklearn.ensemble import RandomForestClassifier >>> from mlxtend.sklearn import EnsembleVoteClassifier >>> clf1 = LogisticRegression(random_seed=1) >>> clf2 = RandomForestClassifier(random_seed=1) >>> clf3 = GaussianNB() >>> X = np.array([[-1, -1], [-2, -1], [-3, -2], [1, 1], [2, 1], [3, 2]]) >>> y = np.array([1, 1, 1, 2, 2, 2]) >>> eclf1 = EnsembleVoteClassifier(clfs=[clf1, clf2, clf3], ... voting='hard', verbose=1) >>> eclf1 = eclf1.fit(X, y) >>> print(eclf1.predict(X)) [1 1 1 2 2 2] >>> eclf2 = EnsembleVoteClassifier(clfs=[clf1, clf2, clf3], voting='soft') >>> eclf2 = eclf2.fit(X, y) >>> print(eclf2.predict(X)) [1 1 1 2 2 2] >>> eclf3 = EnsembleVoteClassifier(clfs=[clf1, clf2, clf3], ... voting='soft', weights=[2,1,1]) >>> eclf3 = eclf3.fit(X, y) >>> print(eclf3.predict(X)) [1 1 1 2 2 2] >>> For more usage examples, please see https://rasbt.github.io/mlxtend/user_guide/classifier/EnsembleVoteClassifier/ ```","title":"EnsembleVoteClassifier"},{"location":"api_subpackages/mlxtend.classifier/#methods_1","text":"fit(X, y, sample_weight=None) Learn weight coefficients from training data for each classifier. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values. sample_weight : array-like, shape = [n_samples], optional Sample weights passed as sample_weights to each regressor in the regressors list as well as the meta_regressor. Raises error if some regressor does not support sample_weight in the fit() method. Returns self : object fit_transform(X, y=None, fit_params) Fit to data, then transform it. Fits transformer to `X` and `y` with optional parameters `fit_params` and returns a transformed version of `X`. Parameters X : array-like of shape (n_samples, n_features) Input samples. y : array-like of shape (n_samples,) or (n_samples, n_outputs), default=None Target values (None for unsupervised transformations). **fit_params : dict Additional fit parameters. Returns X_new : ndarray array of shape (n_samples, n_features_new) Transformed array. get_params(deep=True) Return estimator parameter names for GridSearch support. predict(X) Predict class labels for X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns maj : array-like, shape = [n_samples] Predicted class labels. predict_proba(X) Predict class probabilities for X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns avg : array-like, shape = [n_samples, n_classes] Weighted average probability for each class per sample. score(X, y, sample_weight=None) Return the mean accuracy on the given test data and labels. In multi-label classification, this is the subset accuracy which is a harsh metric since you require for each sample that each label set be correctly predicted. Parameters X : array-like of shape (n_samples, n_features) Test samples. y : array-like of shape (n_samples,) or (n_samples, n_outputs) True labels for X . sample_weight : array-like of shape (n_samples,), default=None Sample weights. Returns score : float Mean accuracy of self.predict(X) w.r.t. y . set_output( , transform=None)* Set output container. See :ref:`sphx_glr_auto_examples_miscellaneous_plot_set_output.py` for an example on how to use the API. Parameters transform : {\"default\", \"pandas\"}, default=None Configure output of transform and fit_transform . \"default\" : Default output format of a transformer \"pandas\" : DataFrame output None : Transform configuration is unchanged Returns self : estimator instance Estimator instance. set_params( params) Set the parameters of this estimator. The method works on simple estimators as well as on nested objects (such as :class:`~sklearn.pipeline.Pipeline`). The latter have parameters of the form ``<component>__<parameter>`` so that it's possible to update each component of a nested object. Parameters **params : dict Estimator parameters. Returns self : estimator instance Estimator instance. transform(X) Return class labels or probabilities for X for each estimator. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns If voting='soft'`` : array-like = [n_classifiers, n_samples, n_classes] Class probabilties calculated by each classifier. If voting='hard'`` : array-like = [n_classifiers, n_samples] Class labels predicted by each classifier.","title":"Methods"},{"location":"api_subpackages/mlxtend.classifier/#logisticregression","text":"LogisticRegression(eta=0.01, epochs=50, l2_lambda=0.0, minibatches=1, random_seed=None, print_progress=0) Logistic regression classifier. Note that this implementation of Logistic Regression expects binary class labels in {0, 1}. Parameters eta : float (default: 0.01) Learning rate (between 0.0 and 1.0) epochs : int (default: 50) Passes over the training dataset. Prior to each epoch, the dataset is shuffled if minibatches > 1 to prevent cycles in stochastic gradient descent. l2_lambda : float Regularization parameter for L2 regularization. No regularization if l2_lambda=0.0. minibatches : int (default: 1) The number of minibatches for gradient-based optimization. If 1: Gradient Descent learning If len(y): Stochastic Gradient Descent (SGD) online learning If 1 < minibatches < len(y): SGD Minibatch learning random_seed : int (default: None) Set random state for shuffling and initializing the weights. print_progress : int (default: 0) Prints progress in fitting to stderr. 0: No output 1: Epochs elapsed and cost 2: 1 plus time elapsed 3: 2 plus estimated time until completion Attributes w_ : 2d-array, shape={n_features, 1} Model weights after fitting. b_ : 1d-array, shape={1,} Bias unit after fitting. cost_ : list List of floats with cross_entropy cost (sgd or gd) for every epoch. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/classifier/LogisticRegression/","title":"LogisticRegression"},{"location":"api_subpackages/mlxtend.classifier/#methods_2","text":"fit(X, y, init_params=True) Learn model from training data. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values. init_params : bool (default: True) Re-initializes model parameters prior to fitting. Set False to continue training with weights from a previous model fitting. Returns self : object get_params(deep=True) Get parameters for this estimator. Parameters deep : boolean, optional If True, will return the parameters for this estimator and contained subobjects that are estimators. Returns params : mapping of string to any Parameter names mapped to their values.' adapted from https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/base.py Author: Gael Varoquaux gael.varoquaux@normalesup.org License: BSD 3 clause predict(X) Predict targets from X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns target_values : array-like, shape = [n_samples] Predicted target values. predict_proba(X) Predict class probabilities of X from the net input. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns Class 1 probability : float score(X, y) Compute the prediction accuracy Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values (true class labels). Returns acc : float The prediction accuracy as a float between 0.0 and 1.0 (perfect score). set_params( params) Set the parameters of this estimator. The method works on simple estimators as well as on nested objects (such as pipelines). The latter have parameters of the form <component>__<parameter> so that it's possible to update each component of a nested object. Returns self adapted from https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/base.py Author: Gael Varoquaux <gael.varoquaux@normalesup.org> License: BSD 3 clause","title":"Methods"},{"location":"api_subpackages/mlxtend.classifier/#multilayerperceptron","text":"MultiLayerPerceptron(eta=0.5, epochs=50, hidden_layers=[50], n_classes=None, momentum=0.0, l1=0.0, l2=0.0, dropout=1.0, decrease_const=0.0, minibatches=1, random_seed=None, print_progress=0) Multi-layer perceptron classifier with logistic sigmoid activations Parameters eta : float (default: 0.5) Learning rate (between 0.0 and 1.0) epochs : int (default: 50) Passes over the training dataset. Prior to each epoch, the dataset is shuffled if minibatches > 1 to prevent cycles in stochastic gradient descent. hidden_layers : list (default: [50]) Number of units per hidden layer. By default 50 units in the first hidden layer. At the moment only 1 hidden layer is supported n_classes : int (default: None) A positive integer to declare the number of class labels if not all class labels are present in a partial training set. Gets the number of class labels automatically if None. l1 : float (default: 0.0) L1 regularization strength l2 : float (default: 0.0) L2 regularization strength momentum : float (default: 0.0) Momentum constant. Factor multiplied with the gradient of the previous epoch t-1 to improve learning speed w(t) := w(t) - (grad(t) + momentum * grad(t-1)) decrease_const : float (default: 0.0) Decrease constant. Shrinks the learning rate after each epoch via eta / (1 + epoch*decrease_const) minibatches : int (default: 1) Divide the training data into k minibatches for accelerated stochastic gradient descent learning. Gradient Descent Learning if minibatches = 1 Stochastic Gradient Descent learning if minibatches = len(y) Minibatch learning if minibatches > 1 random_seed : int (default: None) Set random state for shuffling and initializing the weights. print_progress : int (default: 0) Prints progress in fitting to stderr. 0: No output 1: Epochs elapsed and cost 2: 1 plus time elapsed 3: 2 plus estimated time until completion Attributes w_ : 2d-array, shape=[n_features, n_classes] Weights after fitting. b_ : 1D-array, shape=[n_classes] Bias units after fitting. cost_ : list List of floats; the mean categorical cross entropy cost after each epoch. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/classifier/MultiLayerPerceptron/","title":"MultiLayerPerceptron"},{"location":"api_subpackages/mlxtend.classifier/#methods_3","text":"fit(X, y, init_params=True) Learn model from training data. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values. init_params : bool (default: True) Re-initializes model parameters prior to fitting. Set False to continue training with weights from a previous model fitting. Returns self : object get_params(deep=True) Get parameters for this estimator. Parameters deep : boolean, optional If True, will return the parameters for this estimator and contained subobjects that are estimators. Returns params : mapping of string to any Parameter names mapped to their values.' adapted from https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/base.py Author: Gael Varoquaux gael.varoquaux@normalesup.org License: BSD 3 clause predict(X) Predict targets from X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns target_values : array-like, shape = [n_samples] Predicted target values. predict_proba(X) Predict class probabilities of X from the net input. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns Class probabilties : array-like, shape= [n_samples, n_classes] score(X, y) Compute the prediction accuracy Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values (true class labels). Returns acc : float The prediction accuracy as a float between 0.0 and 1.0 (perfect score). set_params( params) Set the parameters of this estimator. The method works on simple estimators as well as on nested objects (such as pipelines). The latter have parameters of the form <component>__<parameter> so that it's possible to update each component of a nested object. Returns self adapted from https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/base.py Author: Gael Varoquaux <gael.varoquaux@normalesup.org> License: BSD 3 clause","title":"Methods"},{"location":"api_subpackages/mlxtend.classifier/#onerclassifier","text":"OneRClassifier(resolve_ties='first') OneR (One Rule) Classifier. Parameters resolve_ties : str (default: 'first') Option for how to resolve ties if two or more features have the same error. Options are - 'first' (default): chooses first feature in the list, i.e., feature with the lower column index. - 'chi-squared': performs a chi-squared test for each feature against the target and selects the feature with the lowest p-value. Attributes self.classes_labels_ : array-like, shape = [n_labels] Array containing the unique class labels found in the training set. self.feature_idx_ : int The index of the rules' feature based on the column in the training set. self.p_value_ : float The p value for a given feature. Only available after calling fit when the OneR attribute resolve_ties = 'chi-squared' is set. self.prediction_dict_ : dict Dictionary containing information about the feature's (self.feature_idx_) rules and total error. E.g., {'total error': 37, 'rules (value: class)': {0: 0, 1: 2}} means the total error is 37, and the rules are \"if feature value == 0 classify as 0\" and \"if feature value == 1 classify as 2\". (And classify as class 1 otherwise.) For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/classifier/OneRClassifier/","title":"OneRClassifier"},{"location":"api_subpackages/mlxtend.classifier/#methods_4","text":"fit(X, y) Learn rule from training data. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values. Returns self : object get_params(deep=True) Get parameters for this estimator. Parameters deep : bool, default=True If True, will return the parameters for this estimator and contained subobjects that are estimators. Returns params : dict Parameter names mapped to their values. predict(X) Predict class labels for X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns maj : array-like, shape = [n_samples] Predicted class labels. score(X, y, sample_weight=None) Return the mean accuracy on the given test data and labels. In multi-label classification, this is the subset accuracy which is a harsh metric since you require for each sample that each label set be correctly predicted. Parameters X : array-like of shape (n_samples, n_features) Test samples. y : array-like of shape (n_samples,) or (n_samples, n_outputs) True labels for X . sample_weight : array-like of shape (n_samples,), default=None Sample weights. Returns score : float Mean accuracy of self.predict(X) w.r.t. y . set_params( params) Set the parameters of this estimator. The method works on simple estimators as well as on nested objects (such as :class:`~sklearn.pipeline.Pipeline`). The latter have parameters of the form ``<component>__<parameter>`` so that it's possible to update each component of a nested object. Parameters **params : dict Estimator parameters. Returns self : estimator instance Estimator instance.","title":"Methods"},{"location":"api_subpackages/mlxtend.classifier/#perceptron","text":"Perceptron(eta=0.1, epochs=50, random_seed=None, print_progress=0) Perceptron classifier. Note that this implementation of the Perceptron expects binary class labels in {0, 1}. Parameters eta : float (default: 0.1) Learning rate (between 0.0 and 1.0) epochs : int (default: 50) Number of passes over the training dataset. Prior to each epoch, the dataset is shuffled to prevent cycles. random_seed : int Random state for initializing random weights and shuffling. print_progress : int (default: 0) Prints progress in fitting to stderr. 0: No output 1: Epochs elapsed and cost 2: 1 plus time elapsed 3: 2 plus estimated time until completion Attributes w_ : 2d-array, shape={n_features, 1} Model weights after fitting. b_ : 1d-array, shape={1,} Bias unit after fitting. cost_ : list Number of misclassifications in every epoch. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/classifier/Perceptron/","title":"Perceptron"},{"location":"api_subpackages/mlxtend.classifier/#methods_5","text":"fit(X, y, init_params=True) Learn model from training data. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values. init_params : bool (default: True) Re-initializes model parameters prior to fitting. Set False to continue training with weights from a previous model fitting. Returns self : object get_params(deep=True) Get parameters for this estimator. Parameters deep : boolean, optional If True, will return the parameters for this estimator and contained subobjects that are estimators. Returns params : mapping of string to any Parameter names mapped to their values.' adapted from https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/base.py Author: Gael Varoquaux gael.varoquaux@normalesup.org License: BSD 3 clause predict(X) Predict targets from X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns target_values : array-like, shape = [n_samples] Predicted target values. score(X, y) Compute the prediction accuracy Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values (true class labels). Returns acc : float The prediction accuracy as a float between 0.0 and 1.0 (perfect score). set_params( params) Set the parameters of this estimator. The method works on simple estimators as well as on nested objects (such as pipelines). The latter have parameters of the form <component>__<parameter> so that it's possible to update each component of a nested object. Returns self adapted from https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/base.py Author: Gael Varoquaux <gael.varoquaux@normalesup.org> License: BSD 3 clause","title":"Methods"},{"location":"api_subpackages/mlxtend.classifier/#softmaxregression","text":"SoftmaxRegression(eta=0.01, epochs=50, l2=0.0, minibatches=1, n_classes=None, random_seed=None, print_progress=0) Softmax regression classifier. Parameters eta : float (default: 0.01) Learning rate (between 0.0 and 1.0) epochs : int (default: 50) Passes over the training dataset. Prior to each epoch, the dataset is shuffled if minibatches > 1 to prevent cycles in stochastic gradient descent. l2 : float Regularization parameter for L2 regularization. No regularization if l2=0.0. minibatches : int (default: 1) The number of minibatches for gradient-based optimization. If 1: Gradient Descent learning If len(y): Stochastic Gradient Descent (SGD) online learning If 1 < minibatches < len(y): SGD Minibatch learning n_classes : int (default: None) A positive integer to declare the number of class labels if not all class labels are present in a partial training set. Gets the number of class labels automatically if None. random_seed : int (default: None) Set random state for shuffling and initializing the weights. print_progress : int (default: 0) Prints progress in fitting to stderr. 0: No output 1: Epochs elapsed and cost 2: 1 plus time elapsed 3: 2 plus estimated time until completion Attributes w_ : 2d-array, shape={n_features, 1} Model weights after fitting. b_ : 1d-array, shape={1,} Bias unit after fitting. cost_ : list List of floats, the average cross_entropy for each epoch. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/classifier/SoftmaxRegression/","title":"SoftmaxRegression"},{"location":"api_subpackages/mlxtend.classifier/#methods_6","text":"fit(X, y, init_params=True) Learn model from training data. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values. init_params : bool (default: True) Re-initializes model parameters prior to fitting. Set False to continue training with weights from a previous model fitting. Returns self : object get_params(deep=True) Get parameters for this estimator. Parameters deep : boolean, optional If True, will return the parameters for this estimator and contained subobjects that are estimators. Returns params : mapping of string to any Parameter names mapped to their values.' adapted from https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/base.py Author: Gael Varoquaux gael.varoquaux@normalesup.org License: BSD 3 clause predict(X) Predict targets from X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns target_values : array-like, shape = [n_samples] Predicted target values. predict_proba(X) Predict class probabilities of X from the net input. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns Class probabilties : array-like, shape= [n_samples, n_classes] score(X, y) Compute the prediction accuracy Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values (true class labels). Returns acc : float The prediction accuracy as a float between 0.0 and 1.0 (perfect score). set_params( params) Set the parameters of this estimator. The method works on simple estimators as well as on nested objects (such as pipelines). The latter have parameters of the form <component>__<parameter> so that it's possible to update each component of a nested object. Returns self adapted from https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/base.py Author: Gael Varoquaux <gael.varoquaux@normalesup.org> License: BSD 3 clause","title":"Methods"},{"location":"api_subpackages/mlxtend.classifier/#stackingcvclassifier","text":"StackingCVClassifier(classifiers, meta_classifier, use_probas=False, drop_proba_col=None, cv=2, shuffle=True, random_state=None, stratify=True, verbose=0, use_features_in_secondary=False, store_train_meta_features=False, use_clones=True, n_jobs=None, pre_dispatch='2 n_jobs')* A 'Stacking Cross-Validation' classifier for scikit-learn estimators. New in mlxtend v0.4.3 Parameters classifiers : array-like, shape = [n_classifiers] A list of classifiers. Invoking the fit method on the StackingCVClassifer will fit clones of these original classifiers that will be stored in the class attribute self.clfs_ if use_clones=True . meta_classifier : object The meta-classifier to be fitted on the ensemble of classifiers use_probas : bool (default: False) If True, trains meta-classifier based on predicted probabilities instead of class labels. drop_proba_col : string (default: None) Drops extra \"probability\" column in the feature set, because it is redundant: p(y_c) = 1 - p(y_1) + p(y_2) + ... + p(y_{c-1}). This can be useful for meta-classifiers that are sensitive to perfectly collinear features. If 'last', drops last probability column. If 'first', drops first probability column. Only relevant if use_probas=True . cv : int, cross-validation generator or an iterable, optional (default: 2) Determines the cross-validation splitting strategy. Possible inputs for cv are: - None, to use the default 2-fold cross validation, - integer, to specify the number of folds in a (Stratified)KFold , - An object to be used as a cross-validation generator. - An iterable yielding train, test splits. For integer/None inputs, it will use either a KFold or StratifiedKFold cross validation depending the value of stratify argument. shuffle : bool (default: True) If True, and the cv argument is integer, the training data will be shuffled at fitting stage prior to cross-validation. If the cv argument is a specific cross validation technique, this argument is omitted. random_state : int, RandomState instance or None, optional (default: None) Constrols the randomness of the cv splitter. Used when cv is integer and shuffle=True . New in v0.16.0. stratify : bool (default: True) If True, and the cv argument is integer it will follow a stratified K-Fold cross validation technique. If the cv argument is a specific cross validation technique, this argument is omitted. verbose : int, optional (default=0) Controls the verbosity of the building process. - verbose=0 (default): Prints nothing - verbose=1 : Prints the number & name of the regressor being fitted and which fold is currently being used for fitting - verbose=2 : Prints info about the parameters of the regressor being fitted - verbose>2 : Changes verbose param of the underlying regressor to self.verbose - 2 use_features_in_secondary : bool (default: False) If True, the meta-classifier will be trained both on the predictions of the original classifiers and the original dataset. If False, the meta-classifier will be trained only on the predictions of the original classifiers. store_train_meta_features : bool (default: False) If True, the meta-features computed from the training data used for fitting the meta-classifier stored in the self.train_meta_features_ array, which can be accessed after calling fit . use_clones : bool (default: True) Clones the classifiers for stacking classification if True (default) or else uses the original ones, which will be refitted on the dataset upon calling the fit method. Hence, if use_clones=True, the original input classifiers will remain unmodified upon using the StackingCVClassifier's fit method. Setting use_clones=False is recommended if you are working with estimators that are supporting the scikit-learn fit/predict API interface but are not compatible to scikit-learn's clone function. n_jobs : int or None, optional (default=None) The number of CPUs to use to do the computation. None means 1 unless in a joblib.parallel_backend context. -1 means using all processors. for more details. New in v0.16.0. pre_dispatch : int, or string, optional Controls the number of jobs that get dispatched during parallel execution. Reducing this number can be useful to avoid an explosion of memory consumption when more jobs get dispatched than CPUs can process. This parameter can be: - None, in which case all the jobs are immediately created and spawned. Use this for lightweight and fast-running jobs, to avoid delays due to on-demand spawning of the jobs - An int, giving the exact number of total jobs that are spawned - A string, giving an expression as a function of n_jobs, as in '2*n_jobs' New in v0.16.0. Attributes clfs_ : list, shape=[n_classifiers] Fitted classifiers (clones of the original classifiers) meta_clf_ : estimator Fitted meta-classifier (clone of the original meta-estimator) train_meta_features : numpy array, shape = [n_samples, n_classifiers] meta-features for training data, where n_samples is the number of samples in training data and n_classifiers is the number of classfiers. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/classifier/StackingCVClassifier/","title":"StackingCVClassifier"},{"location":"api_subpackages/mlxtend.classifier/#methods_7","text":"decision_function(X) Predict class confidence scores for X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns scores : shape=(n_samples,) if n_classes == 2 else (n_samples, n_classes). Confidence scores per (sample, class) combination. In the binary case, confidence score for self.classes_[1] where >0 means this class would be predicted. fit(X, y, groups=None, sample_weight=None) Fit ensemble classifers and the meta-classifier. Parameters X : numpy array, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : numpy array, shape = [n_samples] Target values. groups : numpy array/None, shape = [n_samples] The group that each sample belongs to. This is used by specific folding strategies such as GroupKFold() sample_weight : array-like, shape = [n_samples], optional Sample weights passed as sample_weights to each regressor in the regressors list as well as the meta_regressor. Raises error if some regressor does not support sample_weight in the fit() method. Returns self : object fit_transform(X, y=None, fit_params) Fit to data, then transform it. Fits transformer to `X` and `y` with optional parameters `fit_params` and returns a transformed version of `X`. Parameters X : array-like of shape (n_samples, n_features) Input samples. y : array-like of shape (n_samples,) or (n_samples, n_outputs), default=None Target values (None for unsupervised transformations). **fit_params : dict Additional fit parameters. Returns X_new : ndarray array of shape (n_samples, n_features_new) Transformed array. get_params(deep=True) Return estimator parameter names for GridSearch support. predict(X) Predict target values for X. Parameters X : numpy array, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns labels : array-like, shape = [n_samples] Predicted class labels. predict_meta_features(X) Get meta-features of test-data. Parameters X : numpy array, shape = [n_samples, n_features] Test vectors, where n_samples is the number of samples and n_features is the number of features. Returns meta-features : numpy array, shape = [n_samples, n_classifiers] Returns the meta-features for test data. predict_proba(X) Predict class probabilities for X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns proba : array-like, shape = [n_samples, n_classes] or a list of n_outputs of such arrays if n_outputs > 1. Probability for each class per sample. score(X, y, sample_weight=None) Return the mean accuracy on the given test data and labels. In multi-label classification, this is the subset accuracy which is a harsh metric since you require for each sample that each label set be correctly predicted. Parameters X : array-like of shape (n_samples, n_features) Test samples. y : array-like of shape (n_samples,) or (n_samples, n_outputs) True labels for X . sample_weight : array-like of shape (n_samples,), default=None Sample weights. Returns score : float Mean accuracy of self.predict(X) w.r.t. y . set_output( , transform=None)* Set output container. See :ref:`sphx_glr_auto_examples_miscellaneous_plot_set_output.py` for an example on how to use the API. Parameters transform : {\"default\", \"pandas\"}, default=None Configure output of transform and fit_transform . \"default\" : Default output format of a transformer \"pandas\" : DataFrame output None : Transform configuration is unchanged Returns self : estimator instance Estimator instance. set_params( params) Set the parameters of this estimator. Valid parameter keys can be listed with ``get_params()``. Returns self","title":"Methods"},{"location":"api_subpackages/mlxtend.classifier/#properties","text":"named_classifiers None","title":"Properties"},{"location":"api_subpackages/mlxtend.classifier/#stackingclassifier","text":"StackingClassifier(classifiers, meta_classifier, use_probas=False, drop_proba_col=None, average_probas=False, verbose=0, use_features_in_secondary=False, store_train_meta_features=False, use_clones=True, fit_base_estimators=True) A Stacking classifier for scikit-learn estimators for classification. Parameters classifiers : array-like, shape = [n_classifiers] A list of classifiers. Invoking the fit method on the StackingClassifer will fit clones of these original classifiers that will be stored in the class attribute self.clfs_ if use_clones=True (default) and fit_base_estimators=True (default). meta_classifier : object The meta-classifier to be fitted on the ensemble of classifiers use_probas : bool (default: False) If True, trains meta-classifier based on predicted probabilities instead of class labels. drop_proba_col : string (default: None) Drops extra \"probability\" column in the feature set, because it is redundant: p(y_c) = 1 - p(y_1) + p(y_2) + ... + p(y_{c-1}). This can be useful for meta-classifiers that are sensitive to perfectly collinear features. If 'last', drops last probability column. If 'first', drops first probability column. Only relevant if use_probas=True . average_probas : bool (default: False) Averages the probabilities as meta features if True . Only relevant if use_probas=True . verbose : int, optional (default=0) Controls the verbosity of the building process. - verbose=0 (default): Prints nothing - verbose=1 : Prints the number & name of the regressor being fitted - verbose=2 : Prints info about the parameters of the regressor being fitted - verbose>2 : Changes verbose param of the underlying regressor to self.verbose - 2 use_features_in_secondary : bool (default: False) If True, the meta-classifier will be trained both on the predictions of the original classifiers and the original dataset. If False, the meta-classifier will be trained only on the predictions of the original classifiers. store_train_meta_features : bool (default: False) If True, the meta-features computed from the training data used for fitting the meta-classifier stored in the self.train_meta_features_ array, which can be accessed after calling fit . use_clones : bool (default: True) Clones the classifiers for stacking classification if True (default) or else uses the original ones, which will be refitted on the dataset upon calling the fit method. Hence, if use_clones=True, the original input classifiers will remain unmodified upon using the StackingClassifier's fit method. Setting use_clones=False is recommended if you are working with estimators that are supporting the scikit-learn fit/predict API interface but are not compatible to scikit-learn's clone function. fit_base_estimators: bool (default: True) Refits classifiers in classifiers if True; uses references to the classifiers , otherwise (assumes that the classifiers were already fit). Note: fit_base_estimators=False will enforce use_clones to be False, and is incompatible to most scikit-learn wrappers! For instance, if any form of cross-validation is performed this would require the re-fitting classifiers to training folds, which would raise a NotFitterError if fit_base_estimators=False. (New in mlxtend v0.6.) Attributes clfs_ : list, shape=[n_classifiers] Fitted classifiers (clones of the original classifiers) meta_clf_ : estimator Fitted meta-classifier (clone of the original meta-estimator) train_meta_features : numpy array, shape = [n_samples, n_classifiers] meta-features for training data, where n_samples is the number of samples in training data and n_classifiers is the number of classfiers. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/classifier/StackingClassifier/","title":"StackingClassifier"},{"location":"api_subpackages/mlxtend.classifier/#methods_8","text":"decision_function(X) Predict class confidence scores for X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns scores : shape=(n_samples,) if n_classes == 2 else (n_samples, n_classes). Confidence scores per (sample, class) combination. In the binary case, confidence score for self.classes_[1] where >0 means this class would be predicted. fit(X, y, sample_weight=None) Fit ensemble classifers and the meta-classifier. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] or [n_samples, n_outputs] Target values. sample_weight : array-like, shape = [n_samples], optional Sample weights passed as sample_weights to each regressor in the regressors list as well as the meta_regressor. Raises error if some regressor does not support sample_weight in the fit() method. Returns self : object fit_transform(X, y=None, fit_params) Fit to data, then transform it. Fits transformer to `X` and `y` with optional parameters `fit_params` and returns a transformed version of `X`. Parameters X : array-like of shape (n_samples, n_features) Input samples. y : array-like of shape (n_samples,) or (n_samples, n_outputs), default=None Target values (None for unsupervised transformations). **fit_params : dict Additional fit parameters. Returns X_new : ndarray array of shape (n_samples, n_features_new) Transformed array. get_params(deep=True) Return estimator parameter names for GridSearch support. predict(X) Predict target values for X. Parameters X : numpy array, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns labels : array-like, shape = [n_samples] Predicted class labels. predict_meta_features(X) Get meta-features of test-data. Parameters X : numpy array, shape = [n_samples, n_features] Test vectors, where n_samples is the number of samples and n_features is the number of features. Returns meta-features : numpy array, shape = [n_samples, n_classifiers] Returns the meta-features for test data. predict_proba(X) Predict class probabilities for X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns proba : array-like, shape = [n_samples, n_classes] or a list of n_outputs of such arrays if n_outputs > 1. Probability for each class per sample. score(X, y, sample_weight=None) Return the mean accuracy on the given test data and labels. In multi-label classification, this is the subset accuracy which is a harsh metric since you require for each sample that each label set be correctly predicted. Parameters X : array-like of shape (n_samples, n_features) Test samples. y : array-like of shape (n_samples,) or (n_samples, n_outputs) True labels for X . sample_weight : array-like of shape (n_samples,), default=None Sample weights. Returns score : float Mean accuracy of self.predict(X) w.r.t. y . set_output( , transform=None)* Set output container. See :ref:`sphx_glr_auto_examples_miscellaneous_plot_set_output.py` for an example on how to use the API. Parameters transform : {\"default\", \"pandas\"}, default=None Configure output of transform and fit_transform . \"default\" : Default output format of a transformer \"pandas\" : DataFrame output None : Transform configuration is unchanged Returns self : estimator instance Estimator instance. set_params( params) Set the parameters of this estimator. Valid parameter keys can be listed with ``get_params()``. Returns self","title":"Methods"},{"location":"api_subpackages/mlxtend.classifier/#properties_1","text":"named_classifiers None","title":"Properties"},{"location":"api_subpackages/mlxtend.cluster/","text":"mlxtend version: 0.23.1 Kmeans Kmeans(k, max_iter=10, convergence_tolerance=1e-05, random_seed=None, print_progress=0) K-means clustering class. Added in 0.4.1dev Parameters k : int Number of clusters max_iter : int (default: 10) Number of iterations during cluster assignment. Cluster re-assignment stops automatically when the algorithm converged. convergence_tolerance : float (default: 1e-05) Compares current centroids with centroids of the previous iteration using the given tolerance (a small positive float)to determine if the algorithm converged early. random_seed : int (default: None) Set random state for the initial centroid assignment. print_progress : int (default: 0) Prints progress in fitting to stderr. 0: No output 1: Iterations elapsed 2: 1 plus time elapsed 3: 2 plus estimated time until completion Attributes centroids_ : 2d-array, shape={k, n_features} Feature values of the k cluster centroids. custers_ : dictionary The cluster assignments stored as a Python dictionary; the dictionary keys denote the cluster indeces and the items are Python lists of the sample indices that were assigned to each cluster. iterations_ : int Number of iterations until convergence. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/classifier/Kmeans/ Methods fit(X, init_params=True) Learn model from training data. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. init_params : bool (default: True) Re-initializes model parameters prior to fitting. Set False to continue training with weights from a previous model fitting. Returns self : object get_params(deep=True) Get parameters for this estimator. Parameters deep : boolean, optional If True, will return the parameters for this estimator and contained subobjects that are estimators. Returns params : mapping of string to any Parameter names mapped to their values.' adapted from https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/base.py Author: Gael Varoquaux gael.varoquaux@normalesup.org License: BSD 3 clause predict(X) Predict targets from X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns target_values : array-like, shape = [n_samples] Predicted target values. set_params( params) Set the parameters of this estimator. The method works on simple estimators as well as on nested objects (such as pipelines). The latter have parameters of the form <component>__<parameter> so that it's possible to update each component of a nested object. Returns self adapted from https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/base.py Author: Gael Varoquaux <gael.varoquaux@normalesup.org> License: BSD 3 clause","title":"Mlxtend.cluster"},{"location":"api_subpackages/mlxtend.cluster/#kmeans","text":"Kmeans(k, max_iter=10, convergence_tolerance=1e-05, random_seed=None, print_progress=0) K-means clustering class. Added in 0.4.1dev Parameters k : int Number of clusters max_iter : int (default: 10) Number of iterations during cluster assignment. Cluster re-assignment stops automatically when the algorithm converged. convergence_tolerance : float (default: 1e-05) Compares current centroids with centroids of the previous iteration using the given tolerance (a small positive float)to determine if the algorithm converged early. random_seed : int (default: None) Set random state for the initial centroid assignment. print_progress : int (default: 0) Prints progress in fitting to stderr. 0: No output 1: Iterations elapsed 2: 1 plus time elapsed 3: 2 plus estimated time until completion Attributes centroids_ : 2d-array, shape={k, n_features} Feature values of the k cluster centroids. custers_ : dictionary The cluster assignments stored as a Python dictionary; the dictionary keys denote the cluster indeces and the items are Python lists of the sample indices that were assigned to each cluster. iterations_ : int Number of iterations until convergence. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/classifier/Kmeans/","title":"Kmeans"},{"location":"api_subpackages/mlxtend.cluster/#methods","text":"fit(X, init_params=True) Learn model from training data. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. init_params : bool (default: True) Re-initializes model parameters prior to fitting. Set False to continue training with weights from a previous model fitting. Returns self : object get_params(deep=True) Get parameters for this estimator. Parameters deep : boolean, optional If True, will return the parameters for this estimator and contained subobjects that are estimators. Returns params : mapping of string to any Parameter names mapped to their values.' adapted from https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/base.py Author: Gael Varoquaux gael.varoquaux@normalesup.org License: BSD 3 clause predict(X) Predict targets from X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns target_values : array-like, shape = [n_samples] Predicted target values. set_params( params) Set the parameters of this estimator. The method works on simple estimators as well as on nested objects (such as pipelines). The latter have parameters of the form <component>__<parameter> so that it's possible to update each component of a nested object. Returns self adapted from https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/base.py Author: Gael Varoquaux <gael.varoquaux@normalesup.org> License: BSD 3 clause","title":"Methods"},{"location":"api_subpackages/mlxtend.data/","text":"mlxtend version: 0.23.1 autompg_data autompg_data() Auto MPG dataset. Source : https://archive.ics.uci.edu/ml/datasets/Auto+MPG Number of samples : 392 Continuous target variable : mpg Dataset Attributes: 1) cylinders: multi-valued discrete 2) displacement: continuous 3) horsepower: continuous 4) weight: continuous 5) acceleration: continuous 6) model year: multi-valued discrete 7) origin: multi-valued discrete 8) car name: string (unique for each instance) Returns X, y : [n_samples, n_features], [n_targets] X is the feature matrix with 392 auto samples as rows and 8 feature columns (6 rows with NaNs removed). y is a 1-dimensional array of the target MPG values. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/data/autompg_data/ boston_housing_data boston_housing_data() Boston Housing dataset. Source : https://archive.ics.uci.edu/ml/datasets/Housing Number of samples : 506 Continuous target variable : MEDV MEDV = Median value of owner-occupied homes in $1000's Dataset Attributes: 1) CRIM per capita crime rate by town 2) ZN proportion of residential land zoned for lots over 25,000 sq.ft. 3) INDUS proportion of non-retail business acres per town 4) CHAS Charles River dummy variable (= 1 if tract bounds river; 0 otherwise) 5) NOX nitric oxides concentration (parts per 10 million) 6) RM average number of rooms per dwelling 7) AGE proportion of owner-occupied units built prior to 1940 8) DIS weighted distances to five Boston employment centres 9) RAD index of accessibility to radial highways 10) TAX full-value property-tax rate per $10,000 11) PTRATIO pupil-teacher ratio by town 12) B 1000(Bk - 0.63)^2 where Bk is the prop. of b. by town 13) LSTAT % lower status of the population Returns X, y : [n_samples, n_features], [n_class_labels] X is the feature matrix with 506 housing samples as rows and 13 feature columns. y is a 1-dimensional array of the continuous target variable MEDV Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/data/boston_housing_data/ iris_data iris_data(version='uci') Iris flower dataset. Source : https://archive.ics.uci.edu/ml/datasets/Iris Number of samples : 150 Class labels : {0, 1, 2}, distribution: [50, 50, 50] 0 = setosa, 1 = versicolor, 2 = virginica. Dataset Attributes: 1) sepal length [cm] 2) sepal width [cm] 3) petal length [cm] 4) petal width [cm] Parameters version : string, optional (default: 'uci'). Version to use {'uci', 'corrected'}. 'uci' loads the dataset as deposited on the UCI machine learning repository, and 'corrected' provides the version that is consistent with Fisher's original paper. See Note for details. Returns X, y : [n_samples, n_features], [n_class_labels] X is the feature matrix with 150 flower samples as rows, and 4 feature columns sepal length, sepal width, petal length, and petal width. y is a 1-dimensional array of the class labels {0, 1, 2} Note The Iris dataset (originally collected by Edgar Anderson) and available in UCI's machine learning repository is different from the Iris dataset described in the original paper by R.A. Fisher [1]). Precisely, there are two data points (row number 34 and 37) in UCI's Machine Learning repository are different from the origianlly published Iris dataset. Also, the original version of the Iris Dataset, which can be loaded via version='corrected' is the same as the one in R. [1] . A. Fisher (1936). \"The use of multiple measurements in taxonomic problems\". Annals of Eugenics. 7 (2): 179\u2013188 Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/data/iris_data/ loadlocal_mnist loadlocal_mnist(images_path, labels_path) Read MNIST from ubyte files. Parameters images_path : str path to the test or train MNIST ubyte file labels_path : str path to the test or train MNIST class labels file Returns images : [n_samples, n_pixels] numpy.array Pixel values of the images. labels : [n_samples] numpy array Target class labels Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/data/loadlocal_mnist/ make_multiplexer_dataset make_multiplexer_dataset(address_bits=2, sample_size=100, positive_class_ratio=0.5, shuffle=False, random_seed=None) Function to create a binary n-bit multiplexer dataset. New in mlxtend v0.9 Parameters address_bits : int (default: 2) A positive integer that determines the number of address bits in the multiplexer, which in turn determine the n-bit capacity of the multiplexer and therefore the number of features. The number of features is determined by the number of address bits. For example, 2 address bits will result in a 6 bit multiplexer and consequently 6 features (2 + 2^2 = 6). If address_bits=3 , then this results in an 11-bit multiplexer as (2 + 2^3 = 11) with 11 features. sample_size : int (default: 100) The total number of samples generated. positive_class_ratio : float (default: 0.5) The fraction (a float between 0 and 1) of samples in the sample_size d dataset that have class label 1. If positive_class_ratio=0.5 (default), then the ratio of class 0 and class 1 samples is perfectly balanced. shuffle : Bool (default: False) Whether or not to shuffle the features and labels. If False (default), the samples are returned in sorted order starting with sample_size /2 samples with class label 0 and followed by sample_size /2 samples with class label 1. random_seed : int (default: None) Random seed used for generating the multiplexer samples and shuffling. Returns X, y : [n_samples, n_features], [n_class_labels] X is the feature matrix with the number of samples equal to sample_size . The number of features is determined by the number of address bits. For instance, 2 address bits will result in a 6 bit multiplexer and consequently 6 features (2 + 2^2 = 6). All features are binary (values in {0, 1}). y is a 1-dimensional array of class labels in {0, 1}. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/data/make_multiplexer_dataset mnist_data mnist_data() 5000 samples from the MNIST handwritten digits dataset. Data Source : https://yann.lecun.com/exdb/mnist/ Returns X, y : [n_samples, n_features], [n_class_labels] X is the feature matrix with 5000 image samples as rows, each row consists of 28x28 pixels that were unrolled into 784 pixel feature vectors. y contains the 10 unique class labels 0-9. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/data/mnist_data/ three_blobs_data three_blobs_data() A random dataset of 3 2D blobs for clustering. Number of samples : 150 Suggested labels : {0, 1, 2}, distribution: [50, 50, 50] Returns X, y : [n_samples, n_features], [n_cluster_labels] X is the feature matrix with 159 samples as rows and 2 feature columns. y is a 1-dimensional array of the 3 suggested cluster labels 0, 1, 2 Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/data/three_blobs_data wine_data wine_data() Wine dataset. Source : https://archive.ics.uci.edu/ml/datasets/Wine Number of samples : 178 Class labels : {0, 1, 2}, distribution: [59, 71, 48] Dataset Attributes: 1) Alcohol 2) Malic acid 3) Ash 4) Alcalinity of ash 5) Magnesium 6) Total phenols 7) Flavanoids 8) Nonflavanoid phenols 9) Proanthocyanins 10) Color intensity 11) Hue 12) OD280/OD315 of diluted wines 13) Proline Returns X, y : [n_samples, n_features], [n_class_labels] X is the feature matrix with 178 wine samples as rows and 13 feature columns. y is a 1-dimensional array of the 3 class labels 0, 1, 2 Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/data/wine_data","title":"Mlxtend.data"},{"location":"api_subpackages/mlxtend.data/#autompg_data","text":"autompg_data() Auto MPG dataset. Source : https://archive.ics.uci.edu/ml/datasets/Auto+MPG Number of samples : 392 Continuous target variable : mpg Dataset Attributes: 1) cylinders: multi-valued discrete 2) displacement: continuous 3) horsepower: continuous 4) weight: continuous 5) acceleration: continuous 6) model year: multi-valued discrete 7) origin: multi-valued discrete 8) car name: string (unique for each instance) Returns X, y : [n_samples, n_features], [n_targets] X is the feature matrix with 392 auto samples as rows and 8 feature columns (6 rows with NaNs removed). y is a 1-dimensional array of the target MPG values. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/data/autompg_data/","title":"autompg_data"},{"location":"api_subpackages/mlxtend.data/#boston_housing_data","text":"boston_housing_data() Boston Housing dataset. Source : https://archive.ics.uci.edu/ml/datasets/Housing Number of samples : 506 Continuous target variable : MEDV MEDV = Median value of owner-occupied homes in $1000's Dataset Attributes: 1) CRIM per capita crime rate by town 2) ZN proportion of residential land zoned for lots over 25,000 sq.ft. 3) INDUS proportion of non-retail business acres per town 4) CHAS Charles River dummy variable (= 1 if tract bounds river; 0 otherwise) 5) NOX nitric oxides concentration (parts per 10 million) 6) RM average number of rooms per dwelling 7) AGE proportion of owner-occupied units built prior to 1940 8) DIS weighted distances to five Boston employment centres 9) RAD index of accessibility to radial highways 10) TAX full-value property-tax rate per $10,000 11) PTRATIO pupil-teacher ratio by town 12) B 1000(Bk - 0.63)^2 where Bk is the prop. of b. by town 13) LSTAT % lower status of the population Returns X, y : [n_samples, n_features], [n_class_labels] X is the feature matrix with 506 housing samples as rows and 13 feature columns. y is a 1-dimensional array of the continuous target variable MEDV Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/data/boston_housing_data/","title":"boston_housing_data"},{"location":"api_subpackages/mlxtend.data/#iris_data","text":"iris_data(version='uci') Iris flower dataset. Source : https://archive.ics.uci.edu/ml/datasets/Iris Number of samples : 150 Class labels : {0, 1, 2}, distribution: [50, 50, 50] 0 = setosa, 1 = versicolor, 2 = virginica. Dataset Attributes: 1) sepal length [cm] 2) sepal width [cm] 3) petal length [cm] 4) petal width [cm] Parameters version : string, optional (default: 'uci'). Version to use {'uci', 'corrected'}. 'uci' loads the dataset as deposited on the UCI machine learning repository, and 'corrected' provides the version that is consistent with Fisher's original paper. See Note for details. Returns X, y : [n_samples, n_features], [n_class_labels] X is the feature matrix with 150 flower samples as rows, and 4 feature columns sepal length, sepal width, petal length, and petal width. y is a 1-dimensional array of the class labels {0, 1, 2} Note The Iris dataset (originally collected by Edgar Anderson) and available in UCI's machine learning repository is different from the Iris dataset described in the original paper by R.A. Fisher [1]). Precisely, there are two data points (row number 34 and 37) in UCI's Machine Learning repository are different from the origianlly published Iris dataset. Also, the original version of the Iris Dataset, which can be loaded via version='corrected' is the same as the one in R. [1] . A. Fisher (1936). \"The use of multiple measurements in taxonomic problems\". Annals of Eugenics. 7 (2): 179\u2013188 Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/data/iris_data/","title":"iris_data"},{"location":"api_subpackages/mlxtend.data/#loadlocal_mnist","text":"loadlocal_mnist(images_path, labels_path) Read MNIST from ubyte files. Parameters images_path : str path to the test or train MNIST ubyte file labels_path : str path to the test or train MNIST class labels file Returns images : [n_samples, n_pixels] numpy.array Pixel values of the images. labels : [n_samples] numpy array Target class labels Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/data/loadlocal_mnist/","title":"loadlocal_mnist"},{"location":"api_subpackages/mlxtend.data/#make_multiplexer_dataset","text":"make_multiplexer_dataset(address_bits=2, sample_size=100, positive_class_ratio=0.5, shuffle=False, random_seed=None) Function to create a binary n-bit multiplexer dataset. New in mlxtend v0.9 Parameters address_bits : int (default: 2) A positive integer that determines the number of address bits in the multiplexer, which in turn determine the n-bit capacity of the multiplexer and therefore the number of features. The number of features is determined by the number of address bits. For example, 2 address bits will result in a 6 bit multiplexer and consequently 6 features (2 + 2^2 = 6). If address_bits=3 , then this results in an 11-bit multiplexer as (2 + 2^3 = 11) with 11 features. sample_size : int (default: 100) The total number of samples generated. positive_class_ratio : float (default: 0.5) The fraction (a float between 0 and 1) of samples in the sample_size d dataset that have class label 1. If positive_class_ratio=0.5 (default), then the ratio of class 0 and class 1 samples is perfectly balanced. shuffle : Bool (default: False) Whether or not to shuffle the features and labels. If False (default), the samples are returned in sorted order starting with sample_size /2 samples with class label 0 and followed by sample_size /2 samples with class label 1. random_seed : int (default: None) Random seed used for generating the multiplexer samples and shuffling. Returns X, y : [n_samples, n_features], [n_class_labels] X is the feature matrix with the number of samples equal to sample_size . The number of features is determined by the number of address bits. For instance, 2 address bits will result in a 6 bit multiplexer and consequently 6 features (2 + 2^2 = 6). All features are binary (values in {0, 1}). y is a 1-dimensional array of class labels in {0, 1}. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/data/make_multiplexer_dataset","title":"make_multiplexer_dataset"},{"location":"api_subpackages/mlxtend.data/#mnist_data","text":"mnist_data() 5000 samples from the MNIST handwritten digits dataset. Data Source : https://yann.lecun.com/exdb/mnist/ Returns X, y : [n_samples, n_features], [n_class_labels] X is the feature matrix with 5000 image samples as rows, each row consists of 28x28 pixels that were unrolled into 784 pixel feature vectors. y contains the 10 unique class labels 0-9. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/data/mnist_data/","title":"mnist_data"},{"location":"api_subpackages/mlxtend.data/#three_blobs_data","text":"three_blobs_data() A random dataset of 3 2D blobs for clustering. Number of samples : 150 Suggested labels : {0, 1, 2}, distribution: [50, 50, 50] Returns X, y : [n_samples, n_features], [n_cluster_labels] X is the feature matrix with 159 samples as rows and 2 feature columns. y is a 1-dimensional array of the 3 suggested cluster labels 0, 1, 2 Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/data/three_blobs_data","title":"three_blobs_data"},{"location":"api_subpackages/mlxtend.data/#wine_data","text":"wine_data() Wine dataset. Source : https://archive.ics.uci.edu/ml/datasets/Wine Number of samples : 178 Class labels : {0, 1, 2}, distribution: [59, 71, 48] Dataset Attributes: 1) Alcohol 2) Malic acid 3) Ash 4) Alcalinity of ash 5) Magnesium 6) Total phenols 7) Flavanoids 8) Nonflavanoid phenols 9) Proanthocyanins 10) Color intensity 11) Hue 12) OD280/OD315 of diluted wines 13) Proline Returns X, y : [n_samples, n_features], [n_class_labels] X is the feature matrix with 178 wine samples as rows and 13 feature columns. y is a 1-dimensional array of the 3 class labels 0, 1, 2 Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/data/wine_data","title":"wine_data"},{"location":"api_subpackages/mlxtend.evaluate/","text":"mlxtend version: 0.23.1 BootstrapOutOfBag BootstrapOutOfBag(n_splits=200, random_seed=None) Parameters n_splits : int (default=200) Number of bootstrap iterations. Must be larger than 1. random_seed : int (default=None) If int, random_seed is the seed used by the random number generator. Returns train_idx : ndarray The training set indices for that split. test_idx : ndarray The testing set indices for that split. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/BootstrapOutOfBag/ Methods get_n_splits(X=None, y=None, groups=None) Returns the number of splitting iterations in the cross-validator Parameters X : object Always ignored, exists for compatibility with scikit-learn. y : object Always ignored, exists for compatibility with scikit-learn. groups : object Always ignored, exists for compatibility with scikit-learn. Returns n_splits : int Returns the number of splitting iterations in the cross-validator. split(X, y=None, groups=None) y : array-like or None (default: None) Argument is not used and only included as parameter for compatibility, similar to KFold in scikit-learn. groups : array-like or None (default: None) Argument is not used and only included as parameter for compatibility, similar to KFold in scikit-learn. GroupTimeSeriesSplit GroupTimeSeriesSplit(test_size, train_size=None, n_splits=None, gap_size=0, shift_size=1, window_type='rolling') Group time series cross-validator. Parameters test_size : int Size of test dataset. train_size : int (default=None) Size of train dataset. n_splits : int (default=None) Number of the splits. gap_size : int (default=0) Gap size between train and test datasets. shift_size : int (default=1) Step to shift for the next fold. window_type : str (default=\"rolling\") Type of the window. Possible values: \"rolling\", \"expanding\". Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/GroupTimeSeriesSplit/ Methods get_n_splits(X=None, y=None, groups=None) Returns the number of splitting iterations in the cross-validator. Parameters X : object Always ignored, exists for compatibility. y : object Always ignored, exists for compatibility. groups : object Always ignored, exists for compatibility. Returns n_splits : int Returns the number of splitting iterations in the cross-validator. split(X, y=None, groups=None) Generate indices to split data into training and test set. Parameters X : array-like Training data. y : array-like (default=None) Always ignored, exists for compatibility. groups : array-like (default=None) Array with group names or sequence numbers. Yields train : ndarray The training set indices for that split. test : ndarray The testing set indices for that split. PredefinedHoldoutSplit PredefinedHoldoutSplit(valid_indices) Train/Validation set splitter for sklearn's GridSearchCV etc. Uses user-specified train/validation set indices to split a dataset into train/validation sets using user-defined or random indices. Parameters valid_indices : array-like, shape (num_examples,) Indices of the training examples in the training set to be used for validation. All other indices in the training set are used to for a training subset for model fitting. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/PredefinedHoldoutSplit/ Methods get_n_splits(X=None, y=None, groups=None) Returns the number of splitting iterations in the cross-validator Parameters X : object Always ignored, exists for compatibility. y : object Always ignored, exists for compatibility. groups : object Always ignored, exists for compatibility. Returns n_splits : 1 Returns the number of splitting iterations in the cross-validator. Always returns 1. split(X, y, groups=None) Generate indices to split data into training and test set. Parameters X : array-like, shape (num_examples, num_features) Training data, where num_examples is the number of examples and num_features is the number of features. y : array-like, shape (num_examples,) The target variable for supervised learning problems. Stratification is done based on the y labels. groups : object Always ignored, exists for compatibility. Yields train_index : ndarray The training set indices for that split. valid_index : ndarray The validation set indices for that split. RandomHoldoutSplit RandomHoldoutSplit(valid_size=0.5, random_seed=None, stratify=False) Train/Validation set splitter for sklearn's GridSearchCV etc. Provides train/validation set indices to split a dataset into train/validation sets using random indices. Parameters valid_size : float (default: 0.5) Proportion of examples that being assigned as validation examples. 1- valid_size will then automatically be assigned as training set examples. random_seed : int (default: None) The random seed for splitting the data into training and validation set partitions. stratify : bool (default: False) True or False, whether to perform a stratified split or not Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/RandomHoldoutSplit/ Methods get_n_splits(X=None, y=None, groups=None) Returns the number of splitting iterations in the cross-validator Parameters X : object Always ignored, exists for compatibility. y : object Always ignored, exists for compatibility. groups : object Always ignored, exists for compatibility. Returns n_splits : 1 Returns the number of splitting iterations in the cross-validator. Always returns 1. split(X, y, groups=None) Generate indices to split data into training and test set. Parameters X : array-like, shape (num_examples, num_features) Training data, where num_examples is the number of training examples and num_features is the number of features. y : array-like, shape (num_examples,) The target variable for supervised learning problems. Stratification is done based on the y labels. groups : object Always ignored, exists for compatibility. Yields train_index : ndarray The training set indices for that split. valid_index : ndarray The validation set indices for that split. accuracy_score accuracy_score(y_target, y_predicted, method='standard', pos_label=1, normalize=True) General accuracy function for supervised learning. Parameters y_target : array-like, shape=[n_values] True class labels or target values. y_predicted : array-like, shape=[n_values] Predicted class labels or target values. method : str, 'standard' by default. The chosen method for accuracy computation. If set to 'standard', computes overall accuracy. If set to 'binary', computes accuracy for class pos_label. If set to 'average', computes average per-class (balanced) accuracy. If set to 'balanced', computes the scikit-learn-style balanced accuracy. pos_label : str or int, 1 by default. The class whose accuracy score is to be reported. Used only when method is set to 'binary' normalize : bool, True by default. If True, returns fraction of correctly classified samples. If False, returns number of correctly classified samples. Returns score: float Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/accuracy_score/ bias_variance_decomp bias_variance_decomp(estimator, X_train, y_train, X_test, y_test, loss='0-1_loss', num_rounds=200, random_seed=None, fit_params) estimator : object A classifier or regressor object or class implementing both a fit and predict method similar to the scikit-learn API. X_train : array-like, shape=(num_examples, num_features) A training dataset for drawing the bootstrap samples to carry out the bias-variance decomposition. y_train : array-like, shape=(num_examples) Targets (class labels, continuous values in case of regression) associated with the X_train examples. X_test : array-like, shape=(num_examples, num_features) The test dataset for computing the average loss, bias, and variance. y_test : array-like, shape=(num_examples) Targets (class labels, continuous values in case of regression) associated with the X_test examples. loss : str (default='0-1_loss') Loss function for performing the bias-variance decomposition. Currently allowed values are '0-1_loss' and 'mse'. num_rounds : int (default=200) Number of bootstrap rounds (sampling from the training set) for performing the bias-variance decomposition. Each bootstrap sample has the same size as the original training set. random_seed : int (default=None) Random seed for the bootstrap sampling used for the bias-variance decomposition. fit_params : additional parameters Additional parameters to be passed to the .fit() function of the estimator when it is fit to the bootstrap samples. Returns avg_expected_loss, avg_bias, avg_var : returns the average expected average bias, and average bias (all floats), where the average is computed over the data points in the test set. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/bias_variance_decomp/ bootstrap bootstrap(x, func, num_rounds=1000, ci=0.95, ddof=1, seed=None) Implements the ordinary nonparametric bootstrap Parameters x : NumPy array, shape=(n_samples, [n_columns]) An one or multidimensional array of data records func : A function which computes a statistic that is used to compute the bootstrap replicates (the statistic computed from the bootstrap samples). This function must return a scalar value. For example, np.mean or np.median would be an acceptable argument for func if x is a 1-dimensional array or vector. num_rounds : int (default=1000) The number of bootstrap samples to draw where each bootstrap sample has the same number of records as the original dataset. ci : int (default=0.95) An integer in the range (0, 1) that represents the confidence level for computing the confidence interval. For example, ci=0.95 (default) will compute the 95% confidence interval from the bootstrap replicates. ddof : int The delta degrees of freedom used when computing the standard error. seed : int or None (default=None) Random seed for generating bootstrap samples. Returns original, standard_error, (lower_ci, upper_ci) : tuple Returns the statistic of the original sample ( original ), the standard error of the estimate, and the respective confidence interval bounds. Examples ``` >>> from mlxtend.evaluate import bootstrap >>> rng = np.random.RandomState(123) >>> x = rng.normal(loc=5., size=100) >>> original, std_err, ci_bounds = bootstrap(x, ... num_rounds=1000, ... func=np.mean, ... ci=0.95, ... seed=123) >>> print('Mean: %.2f, SE: +/- %.2f, CI95: [%.2f, %.2f]' % (original, ... std_err, ... ci_bounds[0], ... ci_bounds[1])) Mean: 5.03, SE: +/- 0.11, CI95: [4.80, 5.26] >>> For more usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/bootstrap/ ## bootstrap_point632_score *bootstrap_point632_score(estimator, X, y, n_splits=200, method='.632', scoring_func=None, predict_proba=False, random_seed=None, clone_estimator=True, **fit_params)* Implementation of the .632 [1] and .632+ [2] bootstrap for supervised learning References: - [1] Efron, Bradley. 1983. \"Estimating the Error Rate of a Prediction Rule: Improvement on Cross-Validation.\" Journal of the American Statistical Association 78 (382): 316. doi:10.2307/2288636. - [2] Efron, Bradley, and Robert Tibshirani. 1997. \"Improvements on Cross-Validation: The .632+ Bootstrap Method.\" Journal of the American Statistical Association 92 (438): 548. doi:10.2307/2965703. **Parameters** - `estimator` : object An estimator for classification or regression that follows the scikit-learn API and implements \"fit\" and \"predict\" methods. - `X` : array-like The data to fit. Can be, for example a list, or an array at least 2d. - `y` : array-like, optional, default: None The target variable to try to predict in the case of supervised learning. - `n_splits` : int (default=200) Number of bootstrap iterations. Must be larger than 1. - `method` : str (default='.632') The bootstrap method, which can be either - 1) '.632' bootstrap (default) - 2) '.632+' bootstrap - 3) 'oob' (regular out-of-bag, no weighting) for comparison studies. - `scoring_func` : callable, Score function (or loss function) with signature ``scoring_func(y, y_pred, **kwargs)``. If none, uses classification accuracy if the estimator is a classifier and mean squared error if the estimator is a regressor. - `predict_proba` : bool Whether to use the `predict_proba` function for the `estimator` argument. This is to be used in conjunction with `scoring_func` which takes in probability values instead of actual predictions. For example, if the scoring_func is :meth:`sklearn.metrics.roc_auc_score`, then use `predict_proba=True`. Note that this requires `estimator` to have `predict_proba` method implemented. - `random_seed` : int (default=None) If int, random_seed is the seed used by the random number generator. - `clone_estimator` : bool (default=True) Clones the estimator if true, otherwise fits the original. - `fit_params` : additional parameters Additional parameters to be passed to the .fit() function of the estimator when it is fit to the bootstrap samples. **Returns** - `scores` : array of float, shape=(len(list(n_splits)),) Array of scores of the estimator for each bootstrap replicate. **Examples** >>> from sklearn import datasets, linear_model >>> from mlxtend.evaluate import bootstrap_point632_score >>> iris = datasets.load_iris() >>> X = iris.data >>> y = iris.target >>> lr = linear_model.LogisticRegression() >>> scores = bootstrap_point632_score(lr, X, y) >>> acc = np.mean(scores) >>> print('Accuracy:', acc) 0.953023146884 >>> lower = np.percentile(scores, 2.5) >>> upper = np.percentile(scores, 97.5) >>> print('95%% Confidence interval: [%.2f, %.2f]' % (lower, upper)) 95% Confidence interval: [0.90, 0.98] For more usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/bootstrap_point632_score/ ``` cochrans_q cochrans_q(y_target, y_model_predictions)* Cochran's Q test to compare 2 or more models. Parameters y_target : array-like, shape=[n_samples] True class labels as 1D NumPy array. *y_model_predictions : array-likes, shape=[n_samples] Variable number of 2 or more arrays that contain the predicted class labels from models as 1D NumPy array. Returns q, p : float or None, float Returns the Q (chi-squared) value and the p-value Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/cochrans_q/ combined_ftest_5x2cv combined_ftest_5x2cv(estimator1, estimator2, X, y, scoring=None, random_seed=None) Implements the 5x2cv combined F test proposed by Alpaydin 1999, to compare the performance of two models. Parameters estimator1 : scikit-learn classifier or regressor estimator2 : scikit-learn classifier or regressor X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values. scoring : str, callable, or None (default: None) If None (default), uses 'accuracy' for sklearn classifiers and 'r2' for sklearn regressors. If str, uses a sklearn scoring metric string identifier, for example {accuracy, f1, precision, recall, roc_auc} for classifiers, {'mean_absolute_error', 'mean_squared_error'/'neg_mean_squared_error', 'median_absolute_error', 'r2'} for regressors. If a callable object or function is provided, it has to be conform with sklearn's signature scorer(estimator, X, y) ; see https://scikit-learn.org/stable/modules/generated/sklearn.metrics.make_scorer.html for more information. random_seed : int or None (default: None) Random seed for creating the test/train splits. Returns f : float The F-statistic pvalue : float Two-tailed p-value. If the chosen significance level is larger than the p-value, we reject the null hypothesis and accept that there are significant differences in the two compared models. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/combined_ftest_5x2cv/ confusion_matrix confusion_matrix(y_target, y_predicted, binary=False, positive_label=1) Compute a confusion matrix/contingency table. Parameters y_target : array-like, shape=[n_samples] True class labels. y_predicted : array-like, shape=[n_samples] Predicted class labels. binary : bool (default: False) Maps a multi-class problem onto a binary confusion matrix, where the positive class is 1 and all other classes are 0. positive_label : int (default: 1) Class label of the positive class. Returns mat : array-like, shape=[n_classes, n_classes] Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/confusion_matrix/ create_counterfactual create_counterfactual(x_reference, y_desired, model, X_dataset, y_desired_proba=None, lammbda=0.1, random_seed=None) Implementation of the counterfactual method by Wachter et al. 2017 References: - Wachter, S., Mittelstadt, B., & Russell, C. (2017). Counterfactual explanations without opening the black box: Automated decisions and the GDPR. Harv. JL & Tech., 31, 841., https://arxiv.org/abs/1711.00399 Parameters x_reference : array-like, shape=[m_features] The data instance (training example) to be explained. y_desired : int The desired class label for x_reference . model : estimator A (scikit-learn) estimator implementing .predict() and/or predict_proba() . - If model supports predict_proba() , then this is used by default for the first loss term, (lambda * model.predict[_proba](x_counterfact) - y_desired[_proba])^2 - Otherwise, method will fall back to predict . X_dataset : array-like, shape=[n_examples, m_features] A (training) dataset for picking the initial counterfactual as initial value for starting the optimization procedure. y_desired_proba : float (default: None) A float within the range [0, 1] designating the desired class probability for y_desired . - If y_desired_proba=None (default), the first loss term is (lambda * model(x_counterfact) - y_desired)^2 where y_desired is a class label - If y_desired_proba is not None, the first loss term is (lambda * model(x_counterfact) - y_desired_proba)^2 lammbda : Weighting parameter for the first loss term, (lambda * model(x_counterfact) - y_desired[_proba])^2 random_seed : int (default=None) If int, random_seed is the seed used by the random number generator for selecting the inital counterfactual from X_dataset . feature_importance_permutation feature_importance_permutation(X, y, predict_method, metric, num_rounds=1, feature_groups=None, seed=None) Feature importance imputation via permutation importance Parameters X : NumPy array, shape = [n_samples, n_features] Dataset, where n_samples is the number of samples and n_features is the number of features. y : NumPy array, shape = [n_samples] Target values. predict_method : prediction function A callable function that predicts the target values from X. metric : str, callable The metric for evaluating the feature importance through permutation. By default, the strings 'accuracy' is recommended for classifiers and the string 'r2' is recommended for regressors. Optionally, a custom scoring function (e.g., metric=scoring_func ) that accepts two arguments, y_true and y_pred, which have similar shape to the y array. num_rounds : int (default=1) Number of rounds the feature columns are permuted to compute the permutation importance. feature_groups : list or None (default=None) Optional argument for treating certain features as a group. For example [1, 2, [3, 4, 5]] , which can be useful for interpretability, for example, if features 3, 4, 5 are one-hot encoded features. seed : int or None (default=None) Random seed for permuting the feature columns. Returns mean_importance_vals, all_importance_vals : NumPy arrays. The first array, mean_importance_vals has shape [n_features, ] and contains the importance values for all features. The shape of the second array is [n_features, num_rounds] and contains the feature importance for each repetition. If num_rounds=1, it contains the same values as the first array, mean_importance_vals. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/feature_importance_permutation/ ftest ftest(y_target, y_model_predictions)* F-Test test to compare 2 or more models. Parameters y_target : array-like, shape=[n_samples] True class labels as 1D NumPy array. *y_model_predictions : array-likes, shape=[n_samples] Variable number of 2 or more arrays that contain the predicted class labels from models as 1D NumPy array. Returns f, p : float or None, float Returns the F-value and the p-value Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/ftest/ lift_score lift_score(y_target, y_predicted, binary=True, positive_label=1) Lift measures the degree to which the predictions of a classification model are better than randomly-generated predictions. The in terms of True Positives (TP), True Negatives (TN), False Positives (FP), and False Negatives (FN), the lift score is computed as: [ TP / (TP+FP) ] / [ (TP+FN) / (TP+TN+FP+FN) ] Parameters y_target : array-like, shape=[n_samples] True class labels. y_predicted : array-like, shape=[n_samples] Predicted class labels. binary : bool (default: True) Maps a multi-class problem onto a binary, where the positive class is 1 and all other classes are 0. positive_label : int (default: 0) Class label of the positive class. Returns score : float Lift score in the range [0, infinity] Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/lift_score/ mcnemar mcnemar(ary, corrected=True, exact=False) McNemar test for paired nominal data Parameters ary : array-like, shape=[2, 2] 2 x 2 contigency table (as returned by evaluate.mcnemar_table), where a: ary[0, 0]: # of samples that both models predicted correctly b: ary[0, 1]: # of samples that model 1 got right and model 2 got wrong c: ary[1, 0]: # of samples that model 2 got right and model 1 got wrong d: aryCell [1, 1]: # of samples that both models predicted incorrectly corrected : array-like, shape=[n_samples] (default: True) Uses Edward's continuity correction for chi-squared if True exact : bool, (default: False) If True , uses an exact binomial test comparing b to a binomial distribution with n = b + c and p = 0.5. It is highly recommended to use exact=True for sample sizes < 25 since chi-squared is not well-approximated by the chi-squared distribution! Returns chi2, p : float or None, float Returns the chi-squared value and the p-value; if exact=True (default: False ), chi2 is None Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/mcnemar/ mcnemar_table mcnemar_table(y_target, y_model1, y_model2) Compute a 2x2 contigency table for McNemar's test. Parameters y_target : array-like, shape=[n_samples] True class labels as 1D NumPy array. y_model1 : array-like, shape=[n_samples] Predicted class labels from model as 1D NumPy array. y_model2 : array-like, shape=[n_samples] Predicted class labels from model 2 as 1D NumPy array. Returns tb : array-like, shape=[2, 2] 2x2 contingency table with the following contents: a: tb[0, 0]: # of samples that both models predicted correctly b: tb[0, 1]: # of samples that model 1 got right and model 2 got wrong c: tb[1, 0]: # of samples that model 2 got right and model 1 got wrong d: tb[1, 1]: # of samples that both models predicted incorrectly Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/mcnemar_table/ mcnemar_tables mcnemar_tables(y_target, y_model_predictions)* Compute multiple 2x2 contigency tables for McNemar's test or Cochran's Q test. Parameters y_target : array-like, shape=[n_samples] True class labels as 1D NumPy array. y_model_predictions : array-like, shape=[n_samples] Predicted class labels for a model. Returns tables : dict Dictionary of NumPy arrays with shape=[2, 2]. Each dictionary key names the two models to be compared based on the order the models were passed as *y_model_predictions . The number of dictionary entries is equal to the number of pairwise combinations between the m models, i.e., \"m choose 2.\" For example the following target array (containing the true labels) and 3 models y_true = np.array([0, 0, 0, 0, 0, 1, 1, 1, 1, 1]) y_mod0 = np.array([0, 1, 0, 0, 0, 1, 1, 0, 0, 0]) y_mod1 = np.array([0, 0, 1, 1, 0, 1, 1, 0, 0, 0]) y_mod2 = np.array([0, 1, 1, 1, 0, 1, 0, 0, 0, 0]) would result in the following dictionary: {'model_0 vs model_1': array([[ 4., 1.], [ 2., 3.]]), 'model_0 vs model_2': array([[ 3., 0.], [ 3., 4.]]), 'model_1 vs model_2': array([[ 3., 0.], [ 2., 5.]])} Each array is structured in the following way: tb[0, 0]: # of samples that both models predicted correctly tb[0, 1]: # of samples that model a got right and model b got wrong tb[1, 0]: # of samples that model b got right and model a got wrong tb[1, 1]: # of samples that both models predicted incorrectly Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/mcnemar_tables/ paired_ttest_5x2cv paired_ttest_5x2cv(estimator1, estimator2, X, y, scoring=None, random_seed=None) Implements the 5x2cv paired t test proposed by Dieterrich (1998) to compare the performance of two models. Parameters estimator1 : scikit-learn classifier or regressor estimator2 : scikit-learn classifier or regressor X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values. scoring : str, callable, or None (default: None) If None (default), uses 'accuracy' for sklearn classifiers and 'r2' for sklearn regressors. If str, uses a sklearn scoring metric string identifier, for example {accuracy, f1, precision, recall, roc_auc} for classifiers, {'mean_absolute_error', 'mean_squared_error'/'neg_mean_squared_error', 'median_absolute_error', 'r2'} for regressors. If a callable object or function is provided, it has to be conform with sklearn's signature scorer(estimator, X, y) ; see https://scikit-learn.org/stable/modules/generated/sklearn.metrics.make_scorer.html for more information. random_seed : int or None (default: None) Random seed for creating the test/train splits. Returns t : float The t-statistic pvalue : float Two-tailed p-value. If the chosen significance level is larger than the p-value, we reject the null hypothesis and accept that there are significant differences in the two compared models. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/paired_ttest_5x2cv/ paired_ttest_kfold_cv paired_ttest_kfold_cv(estimator1, estimator2, X, y, cv=10, scoring=None, shuffle=False, random_seed=None) Implements the k-fold paired t test procedure to compare the performance of two models. Parameters estimator1 : scikit-learn classifier or regressor estimator2 : scikit-learn classifier or regressor X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values. cv : int (default: 10) Number of splits and iteration for the cross-validation procedure scoring : str, callable, or None (default: None) If None (default), uses 'accuracy' for sklearn classifiers and 'r2' for sklearn regressors. If str, uses a sklearn scoring metric string identifier, for example {accuracy, f1, precision, recall, roc_auc} for classifiers, {'mean_absolute_error', 'mean_squared_error'/'neg_mean_squared_error', 'median_absolute_error', 'r2'} for regressors. If a callable object or function is provided, it has to be conform with sklearn's signature scorer(estimator, X, y) ; see https://scikit-learn.org/stable/modules/generated/sklearn.metrics.make_scorer.html for more information. shuffle : bool (default: True) Whether to shuffle the dataset for generating the k-fold splits. random_seed : int or None (default: None) Random seed for shuffling the dataset for generating the k-fold splits. Ignored if shuffle=False. Returns t : float The t-statistic pvalue : float Two-tailed p-value. If the chosen significance level is larger than the p-value, we reject the null hypothesis and accept that there are significant differences in the two compared models. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/paired_ttest_kfold_cv/ paired_ttest_resampled paired_ttest_resampled(estimator1, estimator2, X, y, num_rounds=30, test_size=0.3, scoring=None, random_seed=None) Implements the resampled paired t test procedure to compare the performance of two models (also called k-hold-out paired t test). Parameters estimator1 : scikit-learn classifier or regressor estimator2 : scikit-learn classifier or regressor X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values. num_rounds : int (default: 30) Number of resampling iterations (i.e., train/test splits) test_size : float or int (default: 0.3) If float, should be between 0.0 and 1.0 and represent the proportion of the dataset to use as a test set. If int, represents the absolute number of test exsamples. scoring : str, callable, or None (default: None) If None (default), uses 'accuracy' for sklearn classifiers and 'r2' for sklearn regressors. If str, uses a sklearn scoring metric string identifier, for example {accuracy, f1, precision, recall, roc_auc} for classifiers, {'mean_absolute_error', 'mean_squared_error'/'neg_mean_squared_error', 'median_absolute_error', 'r2'} for regressors. If a callable object or function is provided, it has to be conform with sklearn's signature scorer(estimator, X, y) ; see https://scikit-learn.org/stable/modules/generated/sklearn.metrics.make_scorer.html for more information. random_seed : int or None (default: None) Random seed for creating the test/train splits. Returns t : float The t-statistic pvalue : float Two-tailed p-value. If the chosen significance level is larger than the p-value, we reject the null hypothesis and accept that there are significant differences in the two compared models. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/paired_ttest_resampled/ permutation_test permutation_test(x, y, func='x_mean != y_mean', method='exact', num_rounds=1000, seed=None, paired=False) Nonparametric permutation test Parameters x : list or numpy array with shape (n_datapoints,) A list or 1D numpy array of the first sample (e.g., the treatment group). y : list or numpy array with shape (n_datapoints,) A list or 1D numpy array of the second sample (e.g., the control group). func : custom function or str (default: 'x_mean != y_mean') function to compute the statistic for the permutation test. - If 'x_mean != y_mean', uses func=lambda x, y: np.abs(np.mean(x) - np.mean(y))) for a two-sided test. - If 'x_mean > y_mean', uses func=lambda x, y: np.mean(x) - np.mean(y)) for a one-sided test. - If 'x_mean < y_mean', uses func=lambda x, y: np.mean(y) - np.mean(x)) for a one-sided test. method : 'approximate' or 'exact' (default: 'exact') If 'exact' (default), all possible permutations are considered. If 'approximate' the number of drawn samples is given by num_rounds . Note that 'exact' is typically not feasible unless the dataset size is relatively small. paired : bool If True, a paired test is performed by only exchanging each datapoint with its associate. num_rounds : int (default: 1000) The number of permutation samples if method='approximate' . seed : int or None (default: None) The random seed for generating permutation samples if method='approximate' . Returns p-value under the null hypothesis Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/permutation_test/ proportion_difference proportion_difference(proportion_1, proportion_2, n_1, n_2=None) Computes the test statistic and p-value for a difference of proportions test. Parameters proportion_1 : float The first proportion proportion_2 : float The second proportion n_1 : int The sample size of the first test sample n_2 : int or None (default=None) The sample size of the second test sample. If None , n_1 = n_2 . Returns z, p : float or None, float Returns the z-score and the p-value Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/proportion_difference/ scoring scoring(y_target, y_predicted, metric='error', positive_label=1, unique_labels='auto') Compute a scoring metric for supervised learning. Parameters y_target : array-like, shape=[n_values] True class labels or target values. y_predicted : array-like, shape=[n_values] Predicted class labels or target values. metric : str (default: 'error') Performance metric: 'accuracy': (TP + TN)/(FP + FN + TP + TN) = 1-ERR 'average per-class accuracy': Average per-class accuracy 'average per-class error': Average per-class error 'balanced per-class accuracy': Average per-class accuracy 'balanced per-class error': Average per-class error 'error': (TP + TN)/(FP+ FN + TP + TN) = 1-ACC 'false_positive_rate': FP/N = FP/(FP + TN) 'true_positive_rate': TP/P = TP/(FN + TP) 'true_negative_rate': TN/N = TN/(FP + TN) 'precision': TP/(TP + FP) 'recall': equal to 'true_positive_rate' 'sensitivity': equal to 'true_positive_rate' or 'recall' 'specificity': equal to 'true_negative_rate' 'f1': 2 * (PRE * REC)/(PRE + REC) 'matthews_corr_coef': (TP TN - FP FN) / (sqrt{(TP + FP)( TP + FN )( TN + FP )( TN + FN )}) Where: [TP: True positives, TN = True negatives, TN: True negatives, FN = False negatives] positive_label : int (default: 1) Label of the positive class for binary classification metrics. unique_labels : str or array-like (default: 'auto') If 'auto', deduces the unique class labels from y_target Returns score : float Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/scoring/","title":"Mlxtend.evaluate"},{"location":"api_subpackages/mlxtend.evaluate/#bootstrapoutofbag","text":"BootstrapOutOfBag(n_splits=200, random_seed=None) Parameters n_splits : int (default=200) Number of bootstrap iterations. Must be larger than 1. random_seed : int (default=None) If int, random_seed is the seed used by the random number generator. Returns train_idx : ndarray The training set indices for that split. test_idx : ndarray The testing set indices for that split. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/BootstrapOutOfBag/","title":"BootstrapOutOfBag"},{"location":"api_subpackages/mlxtend.evaluate/#methods","text":"get_n_splits(X=None, y=None, groups=None) Returns the number of splitting iterations in the cross-validator Parameters X : object Always ignored, exists for compatibility with scikit-learn. y : object Always ignored, exists for compatibility with scikit-learn. groups : object Always ignored, exists for compatibility with scikit-learn. Returns n_splits : int Returns the number of splitting iterations in the cross-validator. split(X, y=None, groups=None) y : array-like or None (default: None) Argument is not used and only included as parameter for compatibility, similar to KFold in scikit-learn. groups : array-like or None (default: None) Argument is not used and only included as parameter for compatibility, similar to KFold in scikit-learn.","title":"Methods"},{"location":"api_subpackages/mlxtend.evaluate/#grouptimeseriessplit","text":"GroupTimeSeriesSplit(test_size, train_size=None, n_splits=None, gap_size=0, shift_size=1, window_type='rolling') Group time series cross-validator. Parameters test_size : int Size of test dataset. train_size : int (default=None) Size of train dataset. n_splits : int (default=None) Number of the splits. gap_size : int (default=0) Gap size between train and test datasets. shift_size : int (default=1) Step to shift for the next fold. window_type : str (default=\"rolling\") Type of the window. Possible values: \"rolling\", \"expanding\". Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/GroupTimeSeriesSplit/","title":"GroupTimeSeriesSplit"},{"location":"api_subpackages/mlxtend.evaluate/#methods_1","text":"get_n_splits(X=None, y=None, groups=None) Returns the number of splitting iterations in the cross-validator. Parameters X : object Always ignored, exists for compatibility. y : object Always ignored, exists for compatibility. groups : object Always ignored, exists for compatibility. Returns n_splits : int Returns the number of splitting iterations in the cross-validator. split(X, y=None, groups=None) Generate indices to split data into training and test set. Parameters X : array-like Training data. y : array-like (default=None) Always ignored, exists for compatibility. groups : array-like (default=None) Array with group names or sequence numbers. Yields train : ndarray The training set indices for that split. test : ndarray The testing set indices for that split.","title":"Methods"},{"location":"api_subpackages/mlxtend.evaluate/#predefinedholdoutsplit","text":"PredefinedHoldoutSplit(valid_indices) Train/Validation set splitter for sklearn's GridSearchCV etc. Uses user-specified train/validation set indices to split a dataset into train/validation sets using user-defined or random indices. Parameters valid_indices : array-like, shape (num_examples,) Indices of the training examples in the training set to be used for validation. All other indices in the training set are used to for a training subset for model fitting. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/PredefinedHoldoutSplit/","title":"PredefinedHoldoutSplit"},{"location":"api_subpackages/mlxtend.evaluate/#methods_2","text":"get_n_splits(X=None, y=None, groups=None) Returns the number of splitting iterations in the cross-validator Parameters X : object Always ignored, exists for compatibility. y : object Always ignored, exists for compatibility. groups : object Always ignored, exists for compatibility. Returns n_splits : 1 Returns the number of splitting iterations in the cross-validator. Always returns 1. split(X, y, groups=None) Generate indices to split data into training and test set. Parameters X : array-like, shape (num_examples, num_features) Training data, where num_examples is the number of examples and num_features is the number of features. y : array-like, shape (num_examples,) The target variable for supervised learning problems. Stratification is done based on the y labels. groups : object Always ignored, exists for compatibility. Yields train_index : ndarray The training set indices for that split. valid_index : ndarray The validation set indices for that split.","title":"Methods"},{"location":"api_subpackages/mlxtend.evaluate/#randomholdoutsplit","text":"RandomHoldoutSplit(valid_size=0.5, random_seed=None, stratify=False) Train/Validation set splitter for sklearn's GridSearchCV etc. Provides train/validation set indices to split a dataset into train/validation sets using random indices. Parameters valid_size : float (default: 0.5) Proportion of examples that being assigned as validation examples. 1- valid_size will then automatically be assigned as training set examples. random_seed : int (default: None) The random seed for splitting the data into training and validation set partitions. stratify : bool (default: False) True or False, whether to perform a stratified split or not Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/RandomHoldoutSplit/","title":"RandomHoldoutSplit"},{"location":"api_subpackages/mlxtend.evaluate/#methods_3","text":"get_n_splits(X=None, y=None, groups=None) Returns the number of splitting iterations in the cross-validator Parameters X : object Always ignored, exists for compatibility. y : object Always ignored, exists for compatibility. groups : object Always ignored, exists for compatibility. Returns n_splits : 1 Returns the number of splitting iterations in the cross-validator. Always returns 1. split(X, y, groups=None) Generate indices to split data into training and test set. Parameters X : array-like, shape (num_examples, num_features) Training data, where num_examples is the number of training examples and num_features is the number of features. y : array-like, shape (num_examples,) The target variable for supervised learning problems. Stratification is done based on the y labels. groups : object Always ignored, exists for compatibility. Yields train_index : ndarray The training set indices for that split. valid_index : ndarray The validation set indices for that split.","title":"Methods"},{"location":"api_subpackages/mlxtend.evaluate/#accuracy_score","text":"accuracy_score(y_target, y_predicted, method='standard', pos_label=1, normalize=True) General accuracy function for supervised learning. Parameters y_target : array-like, shape=[n_values] True class labels or target values. y_predicted : array-like, shape=[n_values] Predicted class labels or target values. method : str, 'standard' by default. The chosen method for accuracy computation. If set to 'standard', computes overall accuracy. If set to 'binary', computes accuracy for class pos_label. If set to 'average', computes average per-class (balanced) accuracy. If set to 'balanced', computes the scikit-learn-style balanced accuracy. pos_label : str or int, 1 by default. The class whose accuracy score is to be reported. Used only when method is set to 'binary' normalize : bool, True by default. If True, returns fraction of correctly classified samples. If False, returns number of correctly classified samples. Returns score: float Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/accuracy_score/","title":"accuracy_score"},{"location":"api_subpackages/mlxtend.evaluate/#bias_variance_decomp","text":"bias_variance_decomp(estimator, X_train, y_train, X_test, y_test, loss='0-1_loss', num_rounds=200, random_seed=None, fit_params) estimator : object A classifier or regressor object or class implementing both a fit and predict method similar to the scikit-learn API. X_train : array-like, shape=(num_examples, num_features) A training dataset for drawing the bootstrap samples to carry out the bias-variance decomposition. y_train : array-like, shape=(num_examples) Targets (class labels, continuous values in case of regression) associated with the X_train examples. X_test : array-like, shape=(num_examples, num_features) The test dataset for computing the average loss, bias, and variance. y_test : array-like, shape=(num_examples) Targets (class labels, continuous values in case of regression) associated with the X_test examples. loss : str (default='0-1_loss') Loss function for performing the bias-variance decomposition. Currently allowed values are '0-1_loss' and 'mse'. num_rounds : int (default=200) Number of bootstrap rounds (sampling from the training set) for performing the bias-variance decomposition. Each bootstrap sample has the same size as the original training set. random_seed : int (default=None) Random seed for the bootstrap sampling used for the bias-variance decomposition. fit_params : additional parameters Additional parameters to be passed to the .fit() function of the estimator when it is fit to the bootstrap samples. Returns avg_expected_loss, avg_bias, avg_var : returns the average expected average bias, and average bias (all floats), where the average is computed over the data points in the test set. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/bias_variance_decomp/","title":"bias_variance_decomp"},{"location":"api_subpackages/mlxtend.evaluate/#bootstrap","text":"bootstrap(x, func, num_rounds=1000, ci=0.95, ddof=1, seed=None) Implements the ordinary nonparametric bootstrap Parameters x : NumPy array, shape=(n_samples, [n_columns]) An one or multidimensional array of data records func : A function which computes a statistic that is used to compute the bootstrap replicates (the statistic computed from the bootstrap samples). This function must return a scalar value. For example, np.mean or np.median would be an acceptable argument for func if x is a 1-dimensional array or vector. num_rounds : int (default=1000) The number of bootstrap samples to draw where each bootstrap sample has the same number of records as the original dataset. ci : int (default=0.95) An integer in the range (0, 1) that represents the confidence level for computing the confidence interval. For example, ci=0.95 (default) will compute the 95% confidence interval from the bootstrap replicates. ddof : int The delta degrees of freedom used when computing the standard error. seed : int or None (default=None) Random seed for generating bootstrap samples. Returns original, standard_error, (lower_ci, upper_ci) : tuple Returns the statistic of the original sample ( original ), the standard error of the estimate, and the respective confidence interval bounds. Examples ``` >>> from mlxtend.evaluate import bootstrap >>> rng = np.random.RandomState(123) >>> x = rng.normal(loc=5., size=100) >>> original, std_err, ci_bounds = bootstrap(x, ... num_rounds=1000, ... func=np.mean, ... ci=0.95, ... seed=123) >>> print('Mean: %.2f, SE: +/- %.2f, CI95: [%.2f, %.2f]' % (original, ... std_err, ... ci_bounds[0], ... ci_bounds[1])) Mean: 5.03, SE: +/- 0.11, CI95: [4.80, 5.26] >>> For more usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/bootstrap/ ## bootstrap_point632_score *bootstrap_point632_score(estimator, X, y, n_splits=200, method='.632', scoring_func=None, predict_proba=False, random_seed=None, clone_estimator=True, **fit_params)* Implementation of the .632 [1] and .632+ [2] bootstrap for supervised learning References: - [1] Efron, Bradley. 1983. \"Estimating the Error Rate of a Prediction Rule: Improvement on Cross-Validation.\" Journal of the American Statistical Association 78 (382): 316. doi:10.2307/2288636. - [2] Efron, Bradley, and Robert Tibshirani. 1997. \"Improvements on Cross-Validation: The .632+ Bootstrap Method.\" Journal of the American Statistical Association 92 (438): 548. doi:10.2307/2965703. **Parameters** - `estimator` : object An estimator for classification or regression that follows the scikit-learn API and implements \"fit\" and \"predict\" methods. - `X` : array-like The data to fit. Can be, for example a list, or an array at least 2d. - `y` : array-like, optional, default: None The target variable to try to predict in the case of supervised learning. - `n_splits` : int (default=200) Number of bootstrap iterations. Must be larger than 1. - `method` : str (default='.632') The bootstrap method, which can be either - 1) '.632' bootstrap (default) - 2) '.632+' bootstrap - 3) 'oob' (regular out-of-bag, no weighting) for comparison studies. - `scoring_func` : callable, Score function (or loss function) with signature ``scoring_func(y, y_pred, **kwargs)``. If none, uses classification accuracy if the estimator is a classifier and mean squared error if the estimator is a regressor. - `predict_proba` : bool Whether to use the `predict_proba` function for the `estimator` argument. This is to be used in conjunction with `scoring_func` which takes in probability values instead of actual predictions. For example, if the scoring_func is :meth:`sklearn.metrics.roc_auc_score`, then use `predict_proba=True`. Note that this requires `estimator` to have `predict_proba` method implemented. - `random_seed` : int (default=None) If int, random_seed is the seed used by the random number generator. - `clone_estimator` : bool (default=True) Clones the estimator if true, otherwise fits the original. - `fit_params` : additional parameters Additional parameters to be passed to the .fit() function of the estimator when it is fit to the bootstrap samples. **Returns** - `scores` : array of float, shape=(len(list(n_splits)),) Array of scores of the estimator for each bootstrap replicate. **Examples** >>> from sklearn import datasets, linear_model >>> from mlxtend.evaluate import bootstrap_point632_score >>> iris = datasets.load_iris() >>> X = iris.data >>> y = iris.target >>> lr = linear_model.LogisticRegression() >>> scores = bootstrap_point632_score(lr, X, y) >>> acc = np.mean(scores) >>> print('Accuracy:', acc) 0.953023146884 >>> lower = np.percentile(scores, 2.5) >>> upper = np.percentile(scores, 97.5) >>> print('95%% Confidence interval: [%.2f, %.2f]' % (lower, upper)) 95% Confidence interval: [0.90, 0.98] For more usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/bootstrap_point632_score/ ```","title":"bootstrap"},{"location":"api_subpackages/mlxtend.evaluate/#cochrans_q","text":"cochrans_q(y_target, y_model_predictions)* Cochran's Q test to compare 2 or more models. Parameters y_target : array-like, shape=[n_samples] True class labels as 1D NumPy array. *y_model_predictions : array-likes, shape=[n_samples] Variable number of 2 or more arrays that contain the predicted class labels from models as 1D NumPy array. Returns q, p : float or None, float Returns the Q (chi-squared) value and the p-value Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/cochrans_q/","title":"cochrans_q"},{"location":"api_subpackages/mlxtend.evaluate/#combined_ftest_5x2cv","text":"combined_ftest_5x2cv(estimator1, estimator2, X, y, scoring=None, random_seed=None) Implements the 5x2cv combined F test proposed by Alpaydin 1999, to compare the performance of two models. Parameters estimator1 : scikit-learn classifier or regressor estimator2 : scikit-learn classifier or regressor X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values. scoring : str, callable, or None (default: None) If None (default), uses 'accuracy' for sklearn classifiers and 'r2' for sklearn regressors. If str, uses a sklearn scoring metric string identifier, for example {accuracy, f1, precision, recall, roc_auc} for classifiers, {'mean_absolute_error', 'mean_squared_error'/'neg_mean_squared_error', 'median_absolute_error', 'r2'} for regressors. If a callable object or function is provided, it has to be conform with sklearn's signature scorer(estimator, X, y) ; see https://scikit-learn.org/stable/modules/generated/sklearn.metrics.make_scorer.html for more information. random_seed : int or None (default: None) Random seed for creating the test/train splits. Returns f : float The F-statistic pvalue : float Two-tailed p-value. If the chosen significance level is larger than the p-value, we reject the null hypothesis and accept that there are significant differences in the two compared models. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/combined_ftest_5x2cv/","title":"combined_ftest_5x2cv"},{"location":"api_subpackages/mlxtend.evaluate/#confusion_matrix","text":"confusion_matrix(y_target, y_predicted, binary=False, positive_label=1) Compute a confusion matrix/contingency table. Parameters y_target : array-like, shape=[n_samples] True class labels. y_predicted : array-like, shape=[n_samples] Predicted class labels. binary : bool (default: False) Maps a multi-class problem onto a binary confusion matrix, where the positive class is 1 and all other classes are 0. positive_label : int (default: 1) Class label of the positive class. Returns mat : array-like, shape=[n_classes, n_classes] Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/confusion_matrix/","title":"confusion_matrix"},{"location":"api_subpackages/mlxtend.evaluate/#create_counterfactual","text":"create_counterfactual(x_reference, y_desired, model, X_dataset, y_desired_proba=None, lammbda=0.1, random_seed=None) Implementation of the counterfactual method by Wachter et al. 2017 References: - Wachter, S., Mittelstadt, B., & Russell, C. (2017). Counterfactual explanations without opening the black box: Automated decisions and the GDPR. Harv. JL & Tech., 31, 841., https://arxiv.org/abs/1711.00399 Parameters x_reference : array-like, shape=[m_features] The data instance (training example) to be explained. y_desired : int The desired class label for x_reference . model : estimator A (scikit-learn) estimator implementing .predict() and/or predict_proba() . - If model supports predict_proba() , then this is used by default for the first loss term, (lambda * model.predict[_proba](x_counterfact) - y_desired[_proba])^2 - Otherwise, method will fall back to predict . X_dataset : array-like, shape=[n_examples, m_features] A (training) dataset for picking the initial counterfactual as initial value for starting the optimization procedure. y_desired_proba : float (default: None) A float within the range [0, 1] designating the desired class probability for y_desired . - If y_desired_proba=None (default), the first loss term is (lambda * model(x_counterfact) - y_desired)^2 where y_desired is a class label - If y_desired_proba is not None, the first loss term is (lambda * model(x_counterfact) - y_desired_proba)^2 lammbda : Weighting parameter for the first loss term, (lambda * model(x_counterfact) - y_desired[_proba])^2 random_seed : int (default=None) If int, random_seed is the seed used by the random number generator for selecting the inital counterfactual from X_dataset .","title":"create_counterfactual"},{"location":"api_subpackages/mlxtend.evaluate/#feature_importance_permutation","text":"feature_importance_permutation(X, y, predict_method, metric, num_rounds=1, feature_groups=None, seed=None) Feature importance imputation via permutation importance Parameters X : NumPy array, shape = [n_samples, n_features] Dataset, where n_samples is the number of samples and n_features is the number of features. y : NumPy array, shape = [n_samples] Target values. predict_method : prediction function A callable function that predicts the target values from X. metric : str, callable The metric for evaluating the feature importance through permutation. By default, the strings 'accuracy' is recommended for classifiers and the string 'r2' is recommended for regressors. Optionally, a custom scoring function (e.g., metric=scoring_func ) that accepts two arguments, y_true and y_pred, which have similar shape to the y array. num_rounds : int (default=1) Number of rounds the feature columns are permuted to compute the permutation importance. feature_groups : list or None (default=None) Optional argument for treating certain features as a group. For example [1, 2, [3, 4, 5]] , which can be useful for interpretability, for example, if features 3, 4, 5 are one-hot encoded features. seed : int or None (default=None) Random seed for permuting the feature columns. Returns mean_importance_vals, all_importance_vals : NumPy arrays. The first array, mean_importance_vals has shape [n_features, ] and contains the importance values for all features. The shape of the second array is [n_features, num_rounds] and contains the feature importance for each repetition. If num_rounds=1, it contains the same values as the first array, mean_importance_vals. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/feature_importance_permutation/","title":"feature_importance_permutation"},{"location":"api_subpackages/mlxtend.evaluate/#ftest","text":"ftest(y_target, y_model_predictions)* F-Test test to compare 2 or more models. Parameters y_target : array-like, shape=[n_samples] True class labels as 1D NumPy array. *y_model_predictions : array-likes, shape=[n_samples] Variable number of 2 or more arrays that contain the predicted class labels from models as 1D NumPy array. Returns f, p : float or None, float Returns the F-value and the p-value Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/ftest/","title":"ftest"},{"location":"api_subpackages/mlxtend.evaluate/#lift_score","text":"lift_score(y_target, y_predicted, binary=True, positive_label=1) Lift measures the degree to which the predictions of a classification model are better than randomly-generated predictions. The in terms of True Positives (TP), True Negatives (TN), False Positives (FP), and False Negatives (FN), the lift score is computed as: [ TP / (TP+FP) ] / [ (TP+FN) / (TP+TN+FP+FN) ] Parameters y_target : array-like, shape=[n_samples] True class labels. y_predicted : array-like, shape=[n_samples] Predicted class labels. binary : bool (default: True) Maps a multi-class problem onto a binary, where the positive class is 1 and all other classes are 0. positive_label : int (default: 0) Class label of the positive class. Returns score : float Lift score in the range [0, infinity] Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/lift_score/","title":"lift_score"},{"location":"api_subpackages/mlxtend.evaluate/#mcnemar","text":"mcnemar(ary, corrected=True, exact=False) McNemar test for paired nominal data Parameters ary : array-like, shape=[2, 2] 2 x 2 contigency table (as returned by evaluate.mcnemar_table), where a: ary[0, 0]: # of samples that both models predicted correctly b: ary[0, 1]: # of samples that model 1 got right and model 2 got wrong c: ary[1, 0]: # of samples that model 2 got right and model 1 got wrong d: aryCell [1, 1]: # of samples that both models predicted incorrectly corrected : array-like, shape=[n_samples] (default: True) Uses Edward's continuity correction for chi-squared if True exact : bool, (default: False) If True , uses an exact binomial test comparing b to a binomial distribution with n = b + c and p = 0.5. It is highly recommended to use exact=True for sample sizes < 25 since chi-squared is not well-approximated by the chi-squared distribution! Returns chi2, p : float or None, float Returns the chi-squared value and the p-value; if exact=True (default: False ), chi2 is None Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/mcnemar/","title":"mcnemar"},{"location":"api_subpackages/mlxtend.evaluate/#mcnemar_table","text":"mcnemar_table(y_target, y_model1, y_model2) Compute a 2x2 contigency table for McNemar's test. Parameters y_target : array-like, shape=[n_samples] True class labels as 1D NumPy array. y_model1 : array-like, shape=[n_samples] Predicted class labels from model as 1D NumPy array. y_model2 : array-like, shape=[n_samples] Predicted class labels from model 2 as 1D NumPy array. Returns tb : array-like, shape=[2, 2] 2x2 contingency table with the following contents: a: tb[0, 0]: # of samples that both models predicted correctly b: tb[0, 1]: # of samples that model 1 got right and model 2 got wrong c: tb[1, 0]: # of samples that model 2 got right and model 1 got wrong d: tb[1, 1]: # of samples that both models predicted incorrectly Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/mcnemar_table/","title":"mcnemar_table"},{"location":"api_subpackages/mlxtend.evaluate/#mcnemar_tables","text":"mcnemar_tables(y_target, y_model_predictions)* Compute multiple 2x2 contigency tables for McNemar's test or Cochran's Q test. Parameters y_target : array-like, shape=[n_samples] True class labels as 1D NumPy array. y_model_predictions : array-like, shape=[n_samples] Predicted class labels for a model. Returns tables : dict Dictionary of NumPy arrays with shape=[2, 2]. Each dictionary key names the two models to be compared based on the order the models were passed as *y_model_predictions . The number of dictionary entries is equal to the number of pairwise combinations between the m models, i.e., \"m choose 2.\" For example the following target array (containing the true labels) and 3 models y_true = np.array([0, 0, 0, 0, 0, 1, 1, 1, 1, 1]) y_mod0 = np.array([0, 1, 0, 0, 0, 1, 1, 0, 0, 0]) y_mod1 = np.array([0, 0, 1, 1, 0, 1, 1, 0, 0, 0]) y_mod2 = np.array([0, 1, 1, 1, 0, 1, 0, 0, 0, 0]) would result in the following dictionary: {'model_0 vs model_1': array([[ 4., 1.], [ 2., 3.]]), 'model_0 vs model_2': array([[ 3., 0.], [ 3., 4.]]), 'model_1 vs model_2': array([[ 3., 0.], [ 2., 5.]])} Each array is structured in the following way: tb[0, 0]: # of samples that both models predicted correctly tb[0, 1]: # of samples that model a got right and model b got wrong tb[1, 0]: # of samples that model b got right and model a got wrong tb[1, 1]: # of samples that both models predicted incorrectly Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/mcnemar_tables/","title":"mcnemar_tables"},{"location":"api_subpackages/mlxtend.evaluate/#paired_ttest_5x2cv","text":"paired_ttest_5x2cv(estimator1, estimator2, X, y, scoring=None, random_seed=None) Implements the 5x2cv paired t test proposed by Dieterrich (1998) to compare the performance of two models. Parameters estimator1 : scikit-learn classifier or regressor estimator2 : scikit-learn classifier or regressor X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values. scoring : str, callable, or None (default: None) If None (default), uses 'accuracy' for sklearn classifiers and 'r2' for sklearn regressors. If str, uses a sklearn scoring metric string identifier, for example {accuracy, f1, precision, recall, roc_auc} for classifiers, {'mean_absolute_error', 'mean_squared_error'/'neg_mean_squared_error', 'median_absolute_error', 'r2'} for regressors. If a callable object or function is provided, it has to be conform with sklearn's signature scorer(estimator, X, y) ; see https://scikit-learn.org/stable/modules/generated/sklearn.metrics.make_scorer.html for more information. random_seed : int or None (default: None) Random seed for creating the test/train splits. Returns t : float The t-statistic pvalue : float Two-tailed p-value. If the chosen significance level is larger than the p-value, we reject the null hypothesis and accept that there are significant differences in the two compared models. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/paired_ttest_5x2cv/","title":"paired_ttest_5x2cv"},{"location":"api_subpackages/mlxtend.evaluate/#paired_ttest_kfold_cv","text":"paired_ttest_kfold_cv(estimator1, estimator2, X, y, cv=10, scoring=None, shuffle=False, random_seed=None) Implements the k-fold paired t test procedure to compare the performance of two models. Parameters estimator1 : scikit-learn classifier or regressor estimator2 : scikit-learn classifier or regressor X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values. cv : int (default: 10) Number of splits and iteration for the cross-validation procedure scoring : str, callable, or None (default: None) If None (default), uses 'accuracy' for sklearn classifiers and 'r2' for sklearn regressors. If str, uses a sklearn scoring metric string identifier, for example {accuracy, f1, precision, recall, roc_auc} for classifiers, {'mean_absolute_error', 'mean_squared_error'/'neg_mean_squared_error', 'median_absolute_error', 'r2'} for regressors. If a callable object or function is provided, it has to be conform with sklearn's signature scorer(estimator, X, y) ; see https://scikit-learn.org/stable/modules/generated/sklearn.metrics.make_scorer.html for more information. shuffle : bool (default: True) Whether to shuffle the dataset for generating the k-fold splits. random_seed : int or None (default: None) Random seed for shuffling the dataset for generating the k-fold splits. Ignored if shuffle=False. Returns t : float The t-statistic pvalue : float Two-tailed p-value. If the chosen significance level is larger than the p-value, we reject the null hypothesis and accept that there are significant differences in the two compared models. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/paired_ttest_kfold_cv/","title":"paired_ttest_kfold_cv"},{"location":"api_subpackages/mlxtend.evaluate/#paired_ttest_resampled","text":"paired_ttest_resampled(estimator1, estimator2, X, y, num_rounds=30, test_size=0.3, scoring=None, random_seed=None) Implements the resampled paired t test procedure to compare the performance of two models (also called k-hold-out paired t test). Parameters estimator1 : scikit-learn classifier or regressor estimator2 : scikit-learn classifier or regressor X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values. num_rounds : int (default: 30) Number of resampling iterations (i.e., train/test splits) test_size : float or int (default: 0.3) If float, should be between 0.0 and 1.0 and represent the proportion of the dataset to use as a test set. If int, represents the absolute number of test exsamples. scoring : str, callable, or None (default: None) If None (default), uses 'accuracy' for sklearn classifiers and 'r2' for sklearn regressors. If str, uses a sklearn scoring metric string identifier, for example {accuracy, f1, precision, recall, roc_auc} for classifiers, {'mean_absolute_error', 'mean_squared_error'/'neg_mean_squared_error', 'median_absolute_error', 'r2'} for regressors. If a callable object or function is provided, it has to be conform with sklearn's signature scorer(estimator, X, y) ; see https://scikit-learn.org/stable/modules/generated/sklearn.metrics.make_scorer.html for more information. random_seed : int or None (default: None) Random seed for creating the test/train splits. Returns t : float The t-statistic pvalue : float Two-tailed p-value. If the chosen significance level is larger than the p-value, we reject the null hypothesis and accept that there are significant differences in the two compared models. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/paired_ttest_resampled/","title":"paired_ttest_resampled"},{"location":"api_subpackages/mlxtend.evaluate/#permutation_test","text":"permutation_test(x, y, func='x_mean != y_mean', method='exact', num_rounds=1000, seed=None, paired=False) Nonparametric permutation test Parameters x : list or numpy array with shape (n_datapoints,) A list or 1D numpy array of the first sample (e.g., the treatment group). y : list or numpy array with shape (n_datapoints,) A list or 1D numpy array of the second sample (e.g., the control group). func : custom function or str (default: 'x_mean != y_mean') function to compute the statistic for the permutation test. - If 'x_mean != y_mean', uses func=lambda x, y: np.abs(np.mean(x) - np.mean(y))) for a two-sided test. - If 'x_mean > y_mean', uses func=lambda x, y: np.mean(x) - np.mean(y)) for a one-sided test. - If 'x_mean < y_mean', uses func=lambda x, y: np.mean(y) - np.mean(x)) for a one-sided test. method : 'approximate' or 'exact' (default: 'exact') If 'exact' (default), all possible permutations are considered. If 'approximate' the number of drawn samples is given by num_rounds . Note that 'exact' is typically not feasible unless the dataset size is relatively small. paired : bool If True, a paired test is performed by only exchanging each datapoint with its associate. num_rounds : int (default: 1000) The number of permutation samples if method='approximate' . seed : int or None (default: None) The random seed for generating permutation samples if method='approximate' . Returns p-value under the null hypothesis Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/permutation_test/","title":"permutation_test"},{"location":"api_subpackages/mlxtend.evaluate/#proportion_difference","text":"proportion_difference(proportion_1, proportion_2, n_1, n_2=None) Computes the test statistic and p-value for a difference of proportions test. Parameters proportion_1 : float The first proportion proportion_2 : float The second proportion n_1 : int The sample size of the first test sample n_2 : int or None (default=None) The sample size of the second test sample. If None , n_1 = n_2 . Returns z, p : float or None, float Returns the z-score and the p-value Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/proportion_difference/","title":"proportion_difference"},{"location":"api_subpackages/mlxtend.evaluate/#scoring","text":"scoring(y_target, y_predicted, metric='error', positive_label=1, unique_labels='auto') Compute a scoring metric for supervised learning. Parameters y_target : array-like, shape=[n_values] True class labels or target values. y_predicted : array-like, shape=[n_values] Predicted class labels or target values. metric : str (default: 'error') Performance metric: 'accuracy': (TP + TN)/(FP + FN + TP + TN) = 1-ERR 'average per-class accuracy': Average per-class accuracy 'average per-class error': Average per-class error 'balanced per-class accuracy': Average per-class accuracy 'balanced per-class error': Average per-class error 'error': (TP + TN)/(FP+ FN + TP + TN) = 1-ACC 'false_positive_rate': FP/N = FP/(FP + TN) 'true_positive_rate': TP/P = TP/(FN + TP) 'true_negative_rate': TN/N = TN/(FP + TN) 'precision': TP/(TP + FP) 'recall': equal to 'true_positive_rate' 'sensitivity': equal to 'true_positive_rate' or 'recall' 'specificity': equal to 'true_negative_rate' 'f1': 2 * (PRE * REC)/(PRE + REC) 'matthews_corr_coef': (TP TN - FP FN) / (sqrt{(TP + FP)( TP + FN )( TN + FP )( TN + FN )}) Where: [TP: True positives, TN = True negatives, TN: True negatives, FN = False negatives] positive_label : int (default: 1) Label of the positive class for binary classification metrics. unique_labels : str or array-like (default: 'auto') If 'auto', deduces the unique class labels from y_target Returns score : float Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/scoring/","title":"scoring"},{"location":"api_subpackages/mlxtend.externals/","text":"mlxtend version: 0.23.1","title":"Mlxtend.externals"},{"location":"api_subpackages/mlxtend.feature_extraction/","text":"mlxtend version: 0.23.1 LinearDiscriminantAnalysis LinearDiscriminantAnalysis(n_discriminants=None) Linear Discriminant Analysis Class Parameters n_discriminants : int (default: None) The number of discrimants for transformation. Keeps the original dimensions of the dataset if None . Attributes w_ : array-like, shape=[n_features, n_discriminants] Projection matrix e_vals_ : array-like, shape=[n_features] Eigenvalues in sorted order. e_vecs_ : array-like, shape=[n_features] Eigenvectors in sorted order. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/feature_extraction/LinearDiscriminantAnalysis/ Methods fit(X, y, n_classes=None) Fit the LDA model with X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values. n_classes : int (default: None) A positive integer to declare the number of class labels if not all class labels are present in a partial training set. Gets the number of class labels automatically if None. Returns self : object get_params(deep=True) Get parameters for this estimator. Parameters deep : boolean, optional If True, will return the parameters for this estimator and contained subobjects that are estimators. Returns params : mapping of string to any Parameter names mapped to their values.' adapted from https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/base.py Author: Gael Varoquaux gael.varoquaux@normalesup.org License: BSD 3 clause set_params( params) Set the parameters of this estimator. The method works on simple estimators as well as on nested objects (such as pipelines). The latter have parameters of the form <component>__<parameter> so that it's possible to update each component of a nested object. Returns self adapted from https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/base.py Author: Gael Varoquaux <gael.varoquaux@normalesup.org> License: BSD 3 clause transform(X) Apply the linear transformation on X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns X_projected : np.ndarray, shape = [n_samples, n_discriminants] Projected training vectors. PrincipalComponentAnalysis PrincipalComponentAnalysis(n_components=None, solver='svd', whitening=False) Principal Component Analysis Class Parameters n_components : int (default: None) The number of principal components for transformation. Keeps the original dimensions of the dataset if None . solver : str (default: 'svd') Method for performing the matrix decomposition. {'eigen', 'svd'} whitening : bool (default: False) Performs whitening such that the covariance matrix of the transformed data will be the identity matrix. Attributes w_ : array-like, shape=[n_features, n_components] Projection matrix e_vals_ : array-like, shape=[n_features] Eigenvalues in sorted order. e_vecs_ : array-like, shape=[n_features] Eigenvectors in sorted order. e_vals_normalized_ : array-like, shape=[n_features] Normalized eigen values such that they sum up to 1. This is equal to what's often referred to as \"explained variance ratios.\" loadings_ : array_like, shape=[n_features, n_features] The factor loadings of the original variables onto the principal components. The columns are the principal components, and the rows are the features loadings. For instance, the first column contains the loadings onto the first principal component. Note that the signs may be flipped depending on whether you use the 'eigen' or 'svd' solver; this does not affect the interpretation of the loadings though. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/feature_extraction/PrincipalComponentAnalysis/ Methods fit(X, y=None) Learn model from training data. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns self : object get_params(deep=True) Get parameters for this estimator. Parameters deep : boolean, optional If True, will return the parameters for this estimator and contained subobjects that are estimators. Returns params : mapping of string to any Parameter names mapped to their values.' adapted from https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/base.py Author: Gael Varoquaux gael.varoquaux@normalesup.org License: BSD 3 clause set_params( params) Set the parameters of this estimator. The method works on simple estimators as well as on nested objects (such as pipelines). The latter have parameters of the form <component>__<parameter> so that it's possible to update each component of a nested object. Returns self adapted from https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/base.py Author: Gael Varoquaux <gael.varoquaux@normalesup.org> License: BSD 3 clause transform(X) Apply the linear transformation on X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns X_projected : np.ndarray, shape = [n_samples, n_components] Projected training vectors. RBFKernelPCA RBFKernelPCA(gamma=15.0, n_components=None, copy_X=True) RBF Kernel Principal Component Analysis for dimensionality reduction. Parameters gamma : float (default: 15.0) Free parameter (coefficient) of the RBF kernel. n_components : int (default: None) The number of principal components for transformation. Keeps the original dimensions of the dataset if None . copy_X : bool (default: True) Copies training data, which is required to compute the projection of new data via the transform method. Uses a reference to X if False. Attributes e_vals_ : array-like, shape=[n_features] Eigenvalues in sorted order. e_vecs_ : array-like, shape=[n_features] Eigenvectors in sorted order. X_projected_ : array-like, shape=[n_samples, n_components] Training samples projected along the component axes. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/feature_extraction/RBFKernelPCA/ Methods fit(X) Learn model from training data. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns self : object get_params(deep=True) Get parameters for this estimator. Parameters deep : boolean, optional If True, will return the parameters for this estimator and contained subobjects that are estimators. Returns params : mapping of string to any Parameter names mapped to their values.' adapted from https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/base.py Author: Gael Varoquaux gael.varoquaux@normalesup.org License: BSD 3 clause set_params( params) Set the parameters of this estimator. The method works on simple estimators as well as on nested objects (such as pipelines). The latter have parameters of the form <component>__<parameter> so that it's possible to update each component of a nested object. Returns self adapted from https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/base.py Author: Gael Varoquaux <gael.varoquaux@normalesup.org> License: BSD 3 clause transform(X) Apply the non-linear transformation on X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns X_projected : np.ndarray, shape = [n_samples, n_components] Projected training vectors.","title":"Mlxtend.feature extraction"},{"location":"api_subpackages/mlxtend.feature_extraction/#lineardiscriminantanalysis","text":"LinearDiscriminantAnalysis(n_discriminants=None) Linear Discriminant Analysis Class Parameters n_discriminants : int (default: None) The number of discrimants for transformation. Keeps the original dimensions of the dataset if None . Attributes w_ : array-like, shape=[n_features, n_discriminants] Projection matrix e_vals_ : array-like, shape=[n_features] Eigenvalues in sorted order. e_vecs_ : array-like, shape=[n_features] Eigenvectors in sorted order. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/feature_extraction/LinearDiscriminantAnalysis/","title":"LinearDiscriminantAnalysis"},{"location":"api_subpackages/mlxtend.feature_extraction/#methods","text":"fit(X, y, n_classes=None) Fit the LDA model with X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values. n_classes : int (default: None) A positive integer to declare the number of class labels if not all class labels are present in a partial training set. Gets the number of class labels automatically if None. Returns self : object get_params(deep=True) Get parameters for this estimator. Parameters deep : boolean, optional If True, will return the parameters for this estimator and contained subobjects that are estimators. Returns params : mapping of string to any Parameter names mapped to their values.' adapted from https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/base.py Author: Gael Varoquaux gael.varoquaux@normalesup.org License: BSD 3 clause set_params( params) Set the parameters of this estimator. The method works on simple estimators as well as on nested objects (such as pipelines). The latter have parameters of the form <component>__<parameter> so that it's possible to update each component of a nested object. Returns self adapted from https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/base.py Author: Gael Varoquaux <gael.varoquaux@normalesup.org> License: BSD 3 clause transform(X) Apply the linear transformation on X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns X_projected : np.ndarray, shape = [n_samples, n_discriminants] Projected training vectors.","title":"Methods"},{"location":"api_subpackages/mlxtend.feature_extraction/#principalcomponentanalysis","text":"PrincipalComponentAnalysis(n_components=None, solver='svd', whitening=False) Principal Component Analysis Class Parameters n_components : int (default: None) The number of principal components for transformation. Keeps the original dimensions of the dataset if None . solver : str (default: 'svd') Method for performing the matrix decomposition. {'eigen', 'svd'} whitening : bool (default: False) Performs whitening such that the covariance matrix of the transformed data will be the identity matrix. Attributes w_ : array-like, shape=[n_features, n_components] Projection matrix e_vals_ : array-like, shape=[n_features] Eigenvalues in sorted order. e_vecs_ : array-like, shape=[n_features] Eigenvectors in sorted order. e_vals_normalized_ : array-like, shape=[n_features] Normalized eigen values such that they sum up to 1. This is equal to what's often referred to as \"explained variance ratios.\" loadings_ : array_like, shape=[n_features, n_features] The factor loadings of the original variables onto the principal components. The columns are the principal components, and the rows are the features loadings. For instance, the first column contains the loadings onto the first principal component. Note that the signs may be flipped depending on whether you use the 'eigen' or 'svd' solver; this does not affect the interpretation of the loadings though. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/feature_extraction/PrincipalComponentAnalysis/","title":"PrincipalComponentAnalysis"},{"location":"api_subpackages/mlxtend.feature_extraction/#methods_1","text":"fit(X, y=None) Learn model from training data. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns self : object get_params(deep=True) Get parameters for this estimator. Parameters deep : boolean, optional If True, will return the parameters for this estimator and contained subobjects that are estimators. Returns params : mapping of string to any Parameter names mapped to their values.' adapted from https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/base.py Author: Gael Varoquaux gael.varoquaux@normalesup.org License: BSD 3 clause set_params( params) Set the parameters of this estimator. The method works on simple estimators as well as on nested objects (such as pipelines). The latter have parameters of the form <component>__<parameter> so that it's possible to update each component of a nested object. Returns self adapted from https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/base.py Author: Gael Varoquaux <gael.varoquaux@normalesup.org> License: BSD 3 clause transform(X) Apply the linear transformation on X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns X_projected : np.ndarray, shape = [n_samples, n_components] Projected training vectors.","title":"Methods"},{"location":"api_subpackages/mlxtend.feature_extraction/#rbfkernelpca","text":"RBFKernelPCA(gamma=15.0, n_components=None, copy_X=True) RBF Kernel Principal Component Analysis for dimensionality reduction. Parameters gamma : float (default: 15.0) Free parameter (coefficient) of the RBF kernel. n_components : int (default: None) The number of principal components for transformation. Keeps the original dimensions of the dataset if None . copy_X : bool (default: True) Copies training data, which is required to compute the projection of new data via the transform method. Uses a reference to X if False. Attributes e_vals_ : array-like, shape=[n_features] Eigenvalues in sorted order. e_vecs_ : array-like, shape=[n_features] Eigenvectors in sorted order. X_projected_ : array-like, shape=[n_samples, n_components] Training samples projected along the component axes. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/feature_extraction/RBFKernelPCA/","title":"RBFKernelPCA"},{"location":"api_subpackages/mlxtend.feature_extraction/#methods_2","text":"fit(X) Learn model from training data. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns self : object get_params(deep=True) Get parameters for this estimator. Parameters deep : boolean, optional If True, will return the parameters for this estimator and contained subobjects that are estimators. Returns params : mapping of string to any Parameter names mapped to their values.' adapted from https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/base.py Author: Gael Varoquaux gael.varoquaux@normalesup.org License: BSD 3 clause set_params( params) Set the parameters of this estimator. The method works on simple estimators as well as on nested objects (such as pipelines). The latter have parameters of the form <component>__<parameter> so that it's possible to update each component of a nested object. Returns self adapted from https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/base.py Author: Gael Varoquaux <gael.varoquaux@normalesup.org> License: BSD 3 clause transform(X) Apply the non-linear transformation on X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns X_projected : np.ndarray, shape = [n_samples, n_components] Projected training vectors.","title":"Methods"},{"location":"api_subpackages/mlxtend.feature_selection/","text":"mlxtend version: 0.23.1 ColumnSelector ColumnSelector(cols=None, drop_axis=False) Object for selecting specific columns from a data set. Parameters cols : array-like (default: None) A list specifying the feature indices to be selected. For example, [1, 4, 5] to select the 2nd, 5th, and 6th feature columns, and ['A','C','D'] to select the name of feature columns A, C and D. If None, returns all columns in the array. drop_axis : bool (default=False) Drops last axis if True and the only one column is selected. This is useful, e.g., when the ColumnSelector is used for selecting only one column and the resulting array should be fed to e.g., a scikit-learn column selector. E.g., instead of returning an array with shape (n_samples, 1), drop_axis=True will return an aray with shape (n_samples,). Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/feature_selection/ColumnSelector/ Methods fit(X, y=None) Mock method. Does nothing. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] (default: None) Returns self fit_transform(X, y=None) Return a slice of the input array. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] (default: None) Returns X_slice : shape = [n_samples, k_features] Subset of the feature space where k_features <= n_features get_params(deep=True) Get parameters for this estimator. Parameters deep : bool, default=True If True, will return the parameters for this estimator and contained subobjects that are estimators. Returns params : dict Parameter names mapped to their values. set_params( params) Set the parameters of this estimator. The method works on simple estimators as well as on nested objects (such as :class:`~sklearn.pipeline.Pipeline`). The latter have parameters of the form ``<component>__<parameter>`` so that it's possible to update each component of a nested object. Parameters **params : dict Estimator parameters. Returns self : estimator instance Estimator instance. transform(X, y=None) Return a slice of the input array. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] (default: None) Returns X_slice : shape = [n_samples, k_features] Subset of the feature space where k_features <= n_features ExhaustiveFeatureSelector ExhaustiveFeatureSelector(estimator, min_features=1, max_features=1, print_progress=True, scoring='accuracy', cv=5, n_jobs=1, pre_dispatch='2 n_jobs', clone_estimator=True, fixed_features=None, feature_groups=None)* Exhaustive Feature Selection for Classification and Regression. (new in v0.4.3) Parameters estimator : scikit-learn classifier or regressor min_features : int (default: 1) Minumum number of features to select max_features : int (default: 1) Maximum number of features to select. If parameter feature_groups is not None, the number of features is equal to the number of feature groups, i.e. len(feature_groups) . For example, if feature_groups = [[0], [1], [2, 3], [4]] , then the max_features value cannot exceed 4. print_progress : bool (default: True) Prints progress as the number of epochs to stderr. scoring : str, (default='accuracy') Scoring metric in {accuracy, f1, precision, recall, roc_auc} for classifiers, {'mean_absolute_error', 'mean_squared_error', 'median_absolute_error', 'r2'} for regressors, or a callable object or function with signature scorer(estimator, X, y) . cv : int (default: 5) Scikit-learn cross-validation generator or int . If estimator is a classifier (or y consists of integer class labels), stratified k-fold is performed, and regular k-fold cross-validation otherwise. No cross-validation if cv is None, False, or 0. n_jobs : int (default: 1) The number of CPUs to use for evaluating different feature subsets in parallel. -1 means 'all CPUs'. pre_dispatch : int, or string (default: '2*n_jobs') Controls the number of jobs that get dispatched during parallel execution if n_jobs > 1 or n_jobs=-1 . Reducing this number can be useful to avoid an explosion of memory consumption when more jobs get dispatched than CPUs can process. This parameter can be: None, in which case all the jobs are immediately created and spawned. Use this for lightweight and fast-running jobs, to avoid delays due to on-demand spawning of the jobs An int, giving the exact number of total jobs that are spawned A string, giving an expression as a function of n_jobs, as in 2*n_jobs clone_estimator : bool (default: True) Clones estimator if True; works with the original estimator instance if False. Set to False if the estimator doesn't implement scikit-learn's set_params and get_params methods. In addition, it is required to set cv=0, and n_jobs=1. fixed_features : tuple (default: None) If not None , the feature indices provided as a tuple will be regarded as fixed by the feature selector. For example, if fixed_features=(1, 3, 7) , the 2nd, 4th, and 8th feature are guaranteed to be present in the solution. Note that if fixed_features is not None , make sure that the number of features to be selected is greater than len(fixed_features) . In other words, ensure that k_features > len(fixed_features) . feature_groups : list or None (default: None) Optional argument for treating certain features as a group. This means, the features within a group are always selected together, never split. For example, feature_groups=[[1], [2], [3, 4, 5]] specifies 3 feature groups.In this case, possible feature selection results with k_features=2 are [[1], [2] , [[1], [3, 4, 5]] , or [[2], [3, 4, 5]] . Feature groups can be useful for interpretability, for example, if features 3, 4, 5 are one-hot encoded features. (For more details, please read the notes at the bottom of this docstring). New in mlxtend v. 0.21.0. Attributes best_idx_ : array-like, shape = [n_predictions] Feature Indices of the selected feature subsets. best_feature_names_ : array-like, shape = [n_predictions] Feature names of the selected feature subsets. If pandas DataFrames are used in the fit method, the feature names correspond to the column names. Otherwise, the feature names are string representation of the feature array indices. New in v 0.13.0. best_score_ : float Cross validation average score of the selected subset. subsets_ : dict A dictionary of selected feature subsets during the exhaustive selection, where the dictionary keys are the lengths k of these feature subsets. The dictionary values are dictionaries themselves with the following keys: 'feature_idx' (tuple of indices of the feature subset) 'feature_names' (tuple of feature names of the feat. subset) 'cv_scores' (list individual cross-validation scores) 'avg_score' (average cross-validation score) Note that if pandas DataFrames are used in the fit method, the 'feature_names' correspond to the column names. Otherwise, the feature names are string representation of the feature array indices. The 'feature_names' is new in v. 0.13.0. Notes (1) If parameter feature_groups is not None, the number of features is equal to the number of feature groups, i.e. len(feature_groups) . For example, if feature_groups = [[0], [1], [2, 3], [4]] , then the max_features value cannot exceed 4. (2) Although two or more individual features may be considered as one group throughout the feature-selection process, it does not mean the individual features of that group have the same impact on the outcome. For instance, in linear regression, the coefficient of the feature 2 and 3 can be different even if they are considered as one group in feature_groups. (3) If both fixed_features and feature_groups are specified, ensure that each feature group contains the fixed_features selection. E.g., for a 3-feature set fixed_features=[0, 1] and feature_groups=[[0, 1], [2]] is valid; fixed_features=[0, 1] and feature_groups=[[0], [1, 2]] is not valid. (4) In case of KeyboardInterrupt, the dictionary subsets may not be completed. If user is still interested in getting the best score, they can use method `finalize_fit`. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/feature_selection/ExhaustiveFeatureSelector/ Methods finalize_fit() None fit(X, y, groups=None, fit_params) Perform feature selection and learn model from training data. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. New in v 0.13.0: pandas DataFrames are now also accepted as argument for X. y : array-like, shape = [n_samples] Target values. groups : array-like, with shape (n_samples,), optional Group labels for the samples used while splitting the dataset into train/test set. Passed to the fit method of the cross-validator. fit_params : dict of string -> object, optional Parameters to pass to to the fit method of classifier. Returns self : object fit_transform(X, y, groups=None, fit_params) Fit to training data and return the best selected features from X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. New in v 0.13.0: pandas DataFrames are now also accepted as argument for X. y : array-like, shape = [n_samples] Target values. groups : array-like, with shape (n_samples,), optional Group labels for the samples used while splitting the dataset into train/test set. Passed to the fit method of the cross-validator. fit_params : dict of string -> object, optional Parameters to pass to to the fit method of classifier. Returns Feature subset of X, shape={n_samples, k_features} get_metric_dict(confidence_interval=0.95) Return metric dictionary Parameters confidence_interval : float (default: 0.95) A positive float between 0.0 and 1.0 to compute the confidence interval bounds of the CV score averages. Returns Dictionary with items where each dictionary value is a list with the number of iterations (number of feature subsets) as its length. The dictionary keys corresponding to these lists are as follows: 'feature_idx': tuple of the indices of the feature subset 'cv_scores': list with individual CV scores 'avg_score': of CV average scores 'std_dev': standard deviation of the CV score average 'std_err': standard error of the CV score average 'ci_bound': confidence interval bound of the CV score average get_params(deep=True) Get parameters for this estimator. Parameters deep : bool, default=True If True, will return the parameters for this estimator and contained subobjects that are estimators. Returns params : dict Parameter names mapped to their values. set_params( params) Set the parameters of this estimator. The method works on simple estimators as well as on nested objects (such as :class:`~sklearn.pipeline.Pipeline`). The latter have parameters of the form ``<component>__<parameter>`` so that it's possible to update each component of a nested object. Parameters **params : dict Estimator parameters. Returns self : estimator instance Estimator instance. transform(X) Return the best selected features from X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. New in v 0.13.0: pandas DataFrames are now also accepted as argument for X. Returns Feature subset of X, shape={n_samples, k_features} SequentialFeatureSelector SequentialFeatureSelector(estimator, k_features=1, forward=True, floating=False, verbose=0, scoring=None, cv=5, n_jobs=1, pre_dispatch='2 n_jobs', clone_estimator=True, fixed_features=None, feature_groups=None)* Sequential Feature Selection for Classification and Regression. Parameters estimator : scikit-learn classifier or regressor k_features : int or tuple or str (default: 1) Number of features to select, where k_features < the full feature set. New in 0.4.2: A tuple containing a min and max value can be provided, and the SFS will consider return any feature combination between min and max that scored highest in cross-validation. For example, the tuple (1, 4) will return any combination from 1 up to 4 features instead of a fixed number of features k. New in 0.8.0: A string argument \"best\" or \"parsimonious\". If \"best\" is provided, the feature selector will return the feature subset with the best cross-validation performance. If \"parsimonious\" is provided as an argument, the smallest feature subset that is within one standard error of the cross-validation performance will be selected. forward : bool (default: True) Forward selection if True, backward selection otherwise floating : bool (default: False) Adds a conditional exclusion/inclusion if True. verbose : int (default: 0), level of verbosity to use in logging. If 0, no output, if 1 number of features in current set, if 2 detailed logging i ncluding timestamp and cv scores at step. scoring : str, callable, or None (default: None) If None (default), uses 'accuracy' for sklearn classifiers and 'r2' for sklearn regressors. If str, uses a sklearn scoring metric string identifier, for example {accuracy, f1, precision, recall, roc_auc} for classifiers, {'mean_absolute_error', 'mean_squared_error'/'neg_mean_squared_error', 'median_absolute_error', 'r2'} for regressors. If a callable object or function is provided, it has to be conform with sklearn's signature scorer(estimator, X, y) ; see https://scikit-learn.org/stable/modules/generated/sklearn.metrics.make_scorer.html for more information. cv : int (default: 5) Integer or iterable yielding train, test splits. If cv is an integer and estimator is a classifier (or y consists of integer class labels) stratified k-fold. Otherwise regular k-fold cross-validation is performed. No cross-validation if cv is None, False, or 0. n_jobs : int (default: 1) The number of CPUs to use for evaluating different feature subsets in parallel. -1 means 'all CPUs'. pre_dispatch : int, or string (default: '2*n_jobs') Controls the number of jobs that get dispatched during parallel execution if n_jobs > 1 or n_jobs=-1 . Reducing this number can be useful to avoid an explosion of memory consumption when more jobs get dispatched than CPUs can process. This parameter can be: None, in which case all the jobs are immediately created and spawned. Use this for lightweight and fast-running jobs, to avoid delays due to on-demand spawning of the jobs An int, giving the exact number of total jobs that are spawned A string, giving an expression as a function of n_jobs, as in 2*n_jobs clone_estimator : bool (default: True) Clones estimator if True; works with the original estimator instance if False. Set to False if the estimator doesn't implement scikit-learn's set_params and get_params methods. In addition, it is required to set cv=0, and n_jobs=1. fixed_features : tuple (default: None) If not None , the feature indices provided as a tuple will be regarded as fixed by the feature selector. For example, if fixed_features=(1, 3, 7) , the 2nd, 4th, and 8th feature are guaranteed to be present in the solution. Note that if fixed_features is not None , make sure that the number of features to be selected is greater than len(fixed_features) . In other words, ensure that k_features > len(fixed_features) . New in mlxtend v. 0.18.0. feature_groups : list or None (default: None) Optional argument for treating certain features as a group. This means, the features within a group are always selected together, never split. For example, feature_groups=[[1], [2], [3, 4, 5]] specifies 3 feature groups. In this case, possible feature selection results with k_features=2 are [[1], [2] , [[1], [3, 4, 5]] , or [[2], [3, 4, 5]] . Feature groups can be useful for interpretability, for example, if features 3, 4, 5 are one-hot encoded features. (For more details, please read the notes at the bottom of this docstring). New in mlxtend v. 0.21.0. Attributes k_feature_idx_ : array-like, shape = [n_predictions] Feature Indices of the selected feature subsets. k_feature_names_ : array-like, shape = [n_predictions] Feature names of the selected feature subsets. If pandas DataFrames are used in the fit method, the feature names correspond to the column names. Otherwise, the feature names are string representation of the feature array indices. New in v 0.13.0. k_score_ : float Cross validation average score of the selected subset. subsets_ : dict A dictionary of selected feature subsets during the sequential selection, where the dictionary keys are the lengths k of these feature subsets. If the parameter feature_groups is not None, the value of key indicates the number of groups that are selected together. The dictionary values are dictionaries themselves with the following keys: 'feature_idx' (tuple of indices of the feature subset) 'feature_names' (tuple of feature names of the feat. subset) 'cv_scores' (list individual cross-validation scores) 'avg_score' (average cross-validation score) Note that if pandas DataFrames are used in the fit method, the 'feature_names' correspond to the column names. Otherwise, the feature names are string representation of the feature array indices. The 'feature_names' is new in v 0.13.0. Notes (1) If parameter feature_groups is not None, the number of features is equal to the number of feature groups, i.e. len(feature_groups) . For example, if feature_groups = [[0], [1], [2, 3], [4]] , then the max_features value cannot exceed 4. (2) Although two or more individual features may be considered as one group throughout the feature-selection process, it does not mean the individual features of that group have the same impact on the outcome. For instance, in linear regression, the coefficient of the feature 2 and 3 can be different even if they are considered as one group in feature_groups. (3) If both fixed_features and feature_groups are specified, ensure that each feature group contains the fixed_features selection. E.g., for a 3-feature set fixed_features=[0, 1] and feature_groups=[[0, 1], [2]] is valid; fixed_features=[0, 1] and feature_groups=[[0], [1, 2]] is not valid. (4) In case of KeyboardInterrupt, the dictionary subsets may not be completed. If user is still interested in getting the best score, they can use method `finalize_fit`. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/feature_selection/SequentialFeatureSelector/ Methods finalize_fit() None fit(X, y, groups=None, fit_params) Perform feature selection and learn model from training data. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. New in v 0.13.0: pandas DataFrames are now also accepted as argument for X. y : array-like, shape = [n_samples] Target values. New in v 0.13.0: pandas DataFrames are now also accepted as argument for y. groups : array-like, with shape (n_samples,), optional Group labels for the samples used while splitting the dataset into train/test set. Passed to the fit method of the cross-validator. fit_params : various, optional Additional parameters that are being passed to the estimator. For example, sample_weights=weights . Returns self : object fit_transform(X, y, groups=None, fit_params) Fit to training data then reduce X to its most important features. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. New in v 0.13.0: pandas DataFrames are now also accepted as argument for X. y : array-like, shape = [n_samples] Target values. New in v 0.13.0: a pandas Series are now also accepted as argument for y. groups : array-like, with shape (n_samples,), optional Group labels for the samples used while splitting the dataset into train/test set. Passed to the fit method of the cross-validator. fit_params : various, optional Additional parameters that are being passed to the estimator. For example, sample_weights=weights . Returns Reduced feature subset of X, shape={n_samples, k_features} generate_error_message_k_features(name) None get_metric_dict(confidence_interval=0.95) Return metric dictionary Parameters confidence_interval : float (default: 0.95) A positive float between 0.0 and 1.0 to compute the confidence interval bounds of the CV score averages. Returns Dictionary with items where each dictionary value is a list with the number of iterations (number of feature subsets) as its length. The dictionary keys corresponding to these lists are as follows: 'feature_idx': tuple of the indices of the feature subset 'cv_scores': list with individual CV scores 'avg_score': of CV average scores 'std_dev': standard deviation of the CV score average 'std_err': standard error of the CV score average 'ci_bound': confidence interval bound of the CV score average get_params(deep=True) Get parameters for this estimator. Parameters deep : bool, default=True If True, will return the parameters for this estimator and contained subobjects that are estimators. Returns params : dict Parameter names mapped to their values. set_params( params) Set the parameters of this estimator. Valid parameter keys can be listed with get_params() . Returns self transform(X) Reduce X to its most important features. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. New in v 0.13.0: pandas DataFrames are now also accepted as argument for X. Returns Reduced feature subset of X, shape={n_samples, k_features} Properties named_estimators Returns List of named estimator tuples, like [('svc', SVC(...))]","title":"Mlxtend.feature selection"},{"location":"api_subpackages/mlxtend.feature_selection/#columnselector","text":"ColumnSelector(cols=None, drop_axis=False) Object for selecting specific columns from a data set. Parameters cols : array-like (default: None) A list specifying the feature indices to be selected. For example, [1, 4, 5] to select the 2nd, 5th, and 6th feature columns, and ['A','C','D'] to select the name of feature columns A, C and D. If None, returns all columns in the array. drop_axis : bool (default=False) Drops last axis if True and the only one column is selected. This is useful, e.g., when the ColumnSelector is used for selecting only one column and the resulting array should be fed to e.g., a scikit-learn column selector. E.g., instead of returning an array with shape (n_samples, 1), drop_axis=True will return an aray with shape (n_samples,). Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/feature_selection/ColumnSelector/","title":"ColumnSelector"},{"location":"api_subpackages/mlxtend.feature_selection/#methods","text":"fit(X, y=None) Mock method. Does nothing. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] (default: None) Returns self fit_transform(X, y=None) Return a slice of the input array. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] (default: None) Returns X_slice : shape = [n_samples, k_features] Subset of the feature space where k_features <= n_features get_params(deep=True) Get parameters for this estimator. Parameters deep : bool, default=True If True, will return the parameters for this estimator and contained subobjects that are estimators. Returns params : dict Parameter names mapped to their values. set_params( params) Set the parameters of this estimator. The method works on simple estimators as well as on nested objects (such as :class:`~sklearn.pipeline.Pipeline`). The latter have parameters of the form ``<component>__<parameter>`` so that it's possible to update each component of a nested object. Parameters **params : dict Estimator parameters. Returns self : estimator instance Estimator instance. transform(X, y=None) Return a slice of the input array. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] (default: None) Returns X_slice : shape = [n_samples, k_features] Subset of the feature space where k_features <= n_features","title":"Methods"},{"location":"api_subpackages/mlxtend.feature_selection/#exhaustivefeatureselector","text":"ExhaustiveFeatureSelector(estimator, min_features=1, max_features=1, print_progress=True, scoring='accuracy', cv=5, n_jobs=1, pre_dispatch='2 n_jobs', clone_estimator=True, fixed_features=None, feature_groups=None)* Exhaustive Feature Selection for Classification and Regression. (new in v0.4.3) Parameters estimator : scikit-learn classifier or regressor min_features : int (default: 1) Minumum number of features to select max_features : int (default: 1) Maximum number of features to select. If parameter feature_groups is not None, the number of features is equal to the number of feature groups, i.e. len(feature_groups) . For example, if feature_groups = [[0], [1], [2, 3], [4]] , then the max_features value cannot exceed 4. print_progress : bool (default: True) Prints progress as the number of epochs to stderr. scoring : str, (default='accuracy') Scoring metric in {accuracy, f1, precision, recall, roc_auc} for classifiers, {'mean_absolute_error', 'mean_squared_error', 'median_absolute_error', 'r2'} for regressors, or a callable object or function with signature scorer(estimator, X, y) . cv : int (default: 5) Scikit-learn cross-validation generator or int . If estimator is a classifier (or y consists of integer class labels), stratified k-fold is performed, and regular k-fold cross-validation otherwise. No cross-validation if cv is None, False, or 0. n_jobs : int (default: 1) The number of CPUs to use for evaluating different feature subsets in parallel. -1 means 'all CPUs'. pre_dispatch : int, or string (default: '2*n_jobs') Controls the number of jobs that get dispatched during parallel execution if n_jobs > 1 or n_jobs=-1 . Reducing this number can be useful to avoid an explosion of memory consumption when more jobs get dispatched than CPUs can process. This parameter can be: None, in which case all the jobs are immediately created and spawned. Use this for lightweight and fast-running jobs, to avoid delays due to on-demand spawning of the jobs An int, giving the exact number of total jobs that are spawned A string, giving an expression as a function of n_jobs, as in 2*n_jobs clone_estimator : bool (default: True) Clones estimator if True; works with the original estimator instance if False. Set to False if the estimator doesn't implement scikit-learn's set_params and get_params methods. In addition, it is required to set cv=0, and n_jobs=1. fixed_features : tuple (default: None) If not None , the feature indices provided as a tuple will be regarded as fixed by the feature selector. For example, if fixed_features=(1, 3, 7) , the 2nd, 4th, and 8th feature are guaranteed to be present in the solution. Note that if fixed_features is not None , make sure that the number of features to be selected is greater than len(fixed_features) . In other words, ensure that k_features > len(fixed_features) . feature_groups : list or None (default: None) Optional argument for treating certain features as a group. This means, the features within a group are always selected together, never split. For example, feature_groups=[[1], [2], [3, 4, 5]] specifies 3 feature groups.In this case, possible feature selection results with k_features=2 are [[1], [2] , [[1], [3, 4, 5]] , or [[2], [3, 4, 5]] . Feature groups can be useful for interpretability, for example, if features 3, 4, 5 are one-hot encoded features. (For more details, please read the notes at the bottom of this docstring). New in mlxtend v. 0.21.0. Attributes best_idx_ : array-like, shape = [n_predictions] Feature Indices of the selected feature subsets. best_feature_names_ : array-like, shape = [n_predictions] Feature names of the selected feature subsets. If pandas DataFrames are used in the fit method, the feature names correspond to the column names. Otherwise, the feature names are string representation of the feature array indices. New in v 0.13.0. best_score_ : float Cross validation average score of the selected subset. subsets_ : dict A dictionary of selected feature subsets during the exhaustive selection, where the dictionary keys are the lengths k of these feature subsets. The dictionary values are dictionaries themselves with the following keys: 'feature_idx' (tuple of indices of the feature subset) 'feature_names' (tuple of feature names of the feat. subset) 'cv_scores' (list individual cross-validation scores) 'avg_score' (average cross-validation score) Note that if pandas DataFrames are used in the fit method, the 'feature_names' correspond to the column names. Otherwise, the feature names are string representation of the feature array indices. The 'feature_names' is new in v. 0.13.0. Notes (1) If parameter feature_groups is not None, the number of features is equal to the number of feature groups, i.e. len(feature_groups) . For example, if feature_groups = [[0], [1], [2, 3], [4]] , then the max_features value cannot exceed 4. (2) Although two or more individual features may be considered as one group throughout the feature-selection process, it does not mean the individual features of that group have the same impact on the outcome. For instance, in linear regression, the coefficient of the feature 2 and 3 can be different even if they are considered as one group in feature_groups. (3) If both fixed_features and feature_groups are specified, ensure that each feature group contains the fixed_features selection. E.g., for a 3-feature set fixed_features=[0, 1] and feature_groups=[[0, 1], [2]] is valid; fixed_features=[0, 1] and feature_groups=[[0], [1, 2]] is not valid. (4) In case of KeyboardInterrupt, the dictionary subsets may not be completed. If user is still interested in getting the best score, they can use method `finalize_fit`. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/feature_selection/ExhaustiveFeatureSelector/","title":"ExhaustiveFeatureSelector"},{"location":"api_subpackages/mlxtend.feature_selection/#methods_1","text":"finalize_fit() None fit(X, y, groups=None, fit_params) Perform feature selection and learn model from training data. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. New in v 0.13.0: pandas DataFrames are now also accepted as argument for X. y : array-like, shape = [n_samples] Target values. groups : array-like, with shape (n_samples,), optional Group labels for the samples used while splitting the dataset into train/test set. Passed to the fit method of the cross-validator. fit_params : dict of string -> object, optional Parameters to pass to to the fit method of classifier. Returns self : object fit_transform(X, y, groups=None, fit_params) Fit to training data and return the best selected features from X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. New in v 0.13.0: pandas DataFrames are now also accepted as argument for X. y : array-like, shape = [n_samples] Target values. groups : array-like, with shape (n_samples,), optional Group labels for the samples used while splitting the dataset into train/test set. Passed to the fit method of the cross-validator. fit_params : dict of string -> object, optional Parameters to pass to to the fit method of classifier. Returns Feature subset of X, shape={n_samples, k_features} get_metric_dict(confidence_interval=0.95) Return metric dictionary Parameters confidence_interval : float (default: 0.95) A positive float between 0.0 and 1.0 to compute the confidence interval bounds of the CV score averages. Returns Dictionary with items where each dictionary value is a list with the number of iterations (number of feature subsets) as its length. The dictionary keys corresponding to these lists are as follows: 'feature_idx': tuple of the indices of the feature subset 'cv_scores': list with individual CV scores 'avg_score': of CV average scores 'std_dev': standard deviation of the CV score average 'std_err': standard error of the CV score average 'ci_bound': confidence interval bound of the CV score average get_params(deep=True) Get parameters for this estimator. Parameters deep : bool, default=True If True, will return the parameters for this estimator and contained subobjects that are estimators. Returns params : dict Parameter names mapped to their values. set_params( params) Set the parameters of this estimator. The method works on simple estimators as well as on nested objects (such as :class:`~sklearn.pipeline.Pipeline`). The latter have parameters of the form ``<component>__<parameter>`` so that it's possible to update each component of a nested object. Parameters **params : dict Estimator parameters. Returns self : estimator instance Estimator instance. transform(X) Return the best selected features from X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. New in v 0.13.0: pandas DataFrames are now also accepted as argument for X. Returns Feature subset of X, shape={n_samples, k_features}","title":"Methods"},{"location":"api_subpackages/mlxtend.feature_selection/#sequentialfeatureselector","text":"SequentialFeatureSelector(estimator, k_features=1, forward=True, floating=False, verbose=0, scoring=None, cv=5, n_jobs=1, pre_dispatch='2 n_jobs', clone_estimator=True, fixed_features=None, feature_groups=None)* Sequential Feature Selection for Classification and Regression. Parameters estimator : scikit-learn classifier or regressor k_features : int or tuple or str (default: 1) Number of features to select, where k_features < the full feature set. New in 0.4.2: A tuple containing a min and max value can be provided, and the SFS will consider return any feature combination between min and max that scored highest in cross-validation. For example, the tuple (1, 4) will return any combination from 1 up to 4 features instead of a fixed number of features k. New in 0.8.0: A string argument \"best\" or \"parsimonious\". If \"best\" is provided, the feature selector will return the feature subset with the best cross-validation performance. If \"parsimonious\" is provided as an argument, the smallest feature subset that is within one standard error of the cross-validation performance will be selected. forward : bool (default: True) Forward selection if True, backward selection otherwise floating : bool (default: False) Adds a conditional exclusion/inclusion if True. verbose : int (default: 0), level of verbosity to use in logging. If 0, no output, if 1 number of features in current set, if 2 detailed logging i ncluding timestamp and cv scores at step. scoring : str, callable, or None (default: None) If None (default), uses 'accuracy' for sklearn classifiers and 'r2' for sklearn regressors. If str, uses a sklearn scoring metric string identifier, for example {accuracy, f1, precision, recall, roc_auc} for classifiers, {'mean_absolute_error', 'mean_squared_error'/'neg_mean_squared_error', 'median_absolute_error', 'r2'} for regressors. If a callable object or function is provided, it has to be conform with sklearn's signature scorer(estimator, X, y) ; see https://scikit-learn.org/stable/modules/generated/sklearn.metrics.make_scorer.html for more information. cv : int (default: 5) Integer or iterable yielding train, test splits. If cv is an integer and estimator is a classifier (or y consists of integer class labels) stratified k-fold. Otherwise regular k-fold cross-validation is performed. No cross-validation if cv is None, False, or 0. n_jobs : int (default: 1) The number of CPUs to use for evaluating different feature subsets in parallel. -1 means 'all CPUs'. pre_dispatch : int, or string (default: '2*n_jobs') Controls the number of jobs that get dispatched during parallel execution if n_jobs > 1 or n_jobs=-1 . Reducing this number can be useful to avoid an explosion of memory consumption when more jobs get dispatched than CPUs can process. This parameter can be: None, in which case all the jobs are immediately created and spawned. Use this for lightweight and fast-running jobs, to avoid delays due to on-demand spawning of the jobs An int, giving the exact number of total jobs that are spawned A string, giving an expression as a function of n_jobs, as in 2*n_jobs clone_estimator : bool (default: True) Clones estimator if True; works with the original estimator instance if False. Set to False if the estimator doesn't implement scikit-learn's set_params and get_params methods. In addition, it is required to set cv=0, and n_jobs=1. fixed_features : tuple (default: None) If not None , the feature indices provided as a tuple will be regarded as fixed by the feature selector. For example, if fixed_features=(1, 3, 7) , the 2nd, 4th, and 8th feature are guaranteed to be present in the solution. Note that if fixed_features is not None , make sure that the number of features to be selected is greater than len(fixed_features) . In other words, ensure that k_features > len(fixed_features) . New in mlxtend v. 0.18.0. feature_groups : list or None (default: None) Optional argument for treating certain features as a group. This means, the features within a group are always selected together, never split. For example, feature_groups=[[1], [2], [3, 4, 5]] specifies 3 feature groups. In this case, possible feature selection results with k_features=2 are [[1], [2] , [[1], [3, 4, 5]] , or [[2], [3, 4, 5]] . Feature groups can be useful for interpretability, for example, if features 3, 4, 5 are one-hot encoded features. (For more details, please read the notes at the bottom of this docstring). New in mlxtend v. 0.21.0. Attributes k_feature_idx_ : array-like, shape = [n_predictions] Feature Indices of the selected feature subsets. k_feature_names_ : array-like, shape = [n_predictions] Feature names of the selected feature subsets. If pandas DataFrames are used in the fit method, the feature names correspond to the column names. Otherwise, the feature names are string representation of the feature array indices. New in v 0.13.0. k_score_ : float Cross validation average score of the selected subset. subsets_ : dict A dictionary of selected feature subsets during the sequential selection, where the dictionary keys are the lengths k of these feature subsets. If the parameter feature_groups is not None, the value of key indicates the number of groups that are selected together. The dictionary values are dictionaries themselves with the following keys: 'feature_idx' (tuple of indices of the feature subset) 'feature_names' (tuple of feature names of the feat. subset) 'cv_scores' (list individual cross-validation scores) 'avg_score' (average cross-validation score) Note that if pandas DataFrames are used in the fit method, the 'feature_names' correspond to the column names. Otherwise, the feature names are string representation of the feature array indices. The 'feature_names' is new in v 0.13.0. Notes (1) If parameter feature_groups is not None, the number of features is equal to the number of feature groups, i.e. len(feature_groups) . For example, if feature_groups = [[0], [1], [2, 3], [4]] , then the max_features value cannot exceed 4. (2) Although two or more individual features may be considered as one group throughout the feature-selection process, it does not mean the individual features of that group have the same impact on the outcome. For instance, in linear regression, the coefficient of the feature 2 and 3 can be different even if they are considered as one group in feature_groups. (3) If both fixed_features and feature_groups are specified, ensure that each feature group contains the fixed_features selection. E.g., for a 3-feature set fixed_features=[0, 1] and feature_groups=[[0, 1], [2]] is valid; fixed_features=[0, 1] and feature_groups=[[0], [1, 2]] is not valid. (4) In case of KeyboardInterrupt, the dictionary subsets may not be completed. If user is still interested in getting the best score, they can use method `finalize_fit`. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/feature_selection/SequentialFeatureSelector/","title":"SequentialFeatureSelector"},{"location":"api_subpackages/mlxtend.feature_selection/#methods_2","text":"finalize_fit() None fit(X, y, groups=None, fit_params) Perform feature selection and learn model from training data. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. New in v 0.13.0: pandas DataFrames are now also accepted as argument for X. y : array-like, shape = [n_samples] Target values. New in v 0.13.0: pandas DataFrames are now also accepted as argument for y. groups : array-like, with shape (n_samples,), optional Group labels for the samples used while splitting the dataset into train/test set. Passed to the fit method of the cross-validator. fit_params : various, optional Additional parameters that are being passed to the estimator. For example, sample_weights=weights . Returns self : object fit_transform(X, y, groups=None, fit_params) Fit to training data then reduce X to its most important features. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. New in v 0.13.0: pandas DataFrames are now also accepted as argument for X. y : array-like, shape = [n_samples] Target values. New in v 0.13.0: a pandas Series are now also accepted as argument for y. groups : array-like, with shape (n_samples,), optional Group labels for the samples used while splitting the dataset into train/test set. Passed to the fit method of the cross-validator. fit_params : various, optional Additional parameters that are being passed to the estimator. For example, sample_weights=weights . Returns Reduced feature subset of X, shape={n_samples, k_features} generate_error_message_k_features(name) None get_metric_dict(confidence_interval=0.95) Return metric dictionary Parameters confidence_interval : float (default: 0.95) A positive float between 0.0 and 1.0 to compute the confidence interval bounds of the CV score averages. Returns Dictionary with items where each dictionary value is a list with the number of iterations (number of feature subsets) as its length. The dictionary keys corresponding to these lists are as follows: 'feature_idx': tuple of the indices of the feature subset 'cv_scores': list with individual CV scores 'avg_score': of CV average scores 'std_dev': standard deviation of the CV score average 'std_err': standard error of the CV score average 'ci_bound': confidence interval bound of the CV score average get_params(deep=True) Get parameters for this estimator. Parameters deep : bool, default=True If True, will return the parameters for this estimator and contained subobjects that are estimators. Returns params : dict Parameter names mapped to their values. set_params( params) Set the parameters of this estimator. Valid parameter keys can be listed with get_params() . Returns self transform(X) Reduce X to its most important features. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. New in v 0.13.0: pandas DataFrames are now also accepted as argument for X. Returns Reduced feature subset of X, shape={n_samples, k_features}","title":"Methods"},{"location":"api_subpackages/mlxtend.feature_selection/#properties","text":"named_estimators Returns List of named estimator tuples, like [('svc', SVC(...))]","title":"Properties"},{"location":"api_subpackages/mlxtend.file_io/","text":"mlxtend version: 0.23.1 find_filegroups find_filegroups(paths, substring='', extensions=None, validity_check=True, ignore_invisible=True, rstrip='', ignore_substring=None) Find and collect files from different directories in a python dictionary. Parameters paths : list Paths of the directories to be searched. Dictionary keys are build from the first directory. substring : str (default: '') Substring that all files have to contain to be considered. extensions : list (default: None) None or list of allowed file extensions for each path. If provided, the number of extensions must match the number of paths . validity_check : bool (default: None) If True , checks if all dictionary values have the same number of file paths. Prints a warning and returns an empty dictionary if the validity check failed. ignore_invisible : bool (default: True) If True , ignores invisible files (i.e., files starting with a period). rstrip : str (default: '') If provided, strips characters from right side of the file base names after splitting the extension. Useful to trim different filenames to a common stem. E.g,. \"abc_d.txt\" and \"abc_d_.csv\" would share the stem \"abc_d\" if rstrip is set to \"_\". ignore_substring : str (default: None) Ignores files that contain the specified substring. Returns groups : dict Dictionary of files paths. Keys are the file names found in the first directory listed in paths (without file extension). Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/file_io/find_filegroups/ find_files find_files(substring, path, recursive=False, check_ext=None, ignore_invisible=True, ignore_substring=None) Find files in a directory based on substring matching. Parameters substring : str Substring of the file to be matched. path : str Path where to look. recursive : bool If true, searches subdirectories recursively. check_ext : str If string (e.g., '.txt'), only returns files that match the specified file extension. ignore_invisible : bool If True , ignores invisible files (i.e., files starting with a period). ignore_substring : str Ignores files that contain the specified substring. Returns results : list List of the matched files. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/file_io/find_files/","title":"Mlxtend.file io"},{"location":"api_subpackages/mlxtend.file_io/#find_filegroups","text":"find_filegroups(paths, substring='', extensions=None, validity_check=True, ignore_invisible=True, rstrip='', ignore_substring=None) Find and collect files from different directories in a python dictionary. Parameters paths : list Paths of the directories to be searched. Dictionary keys are build from the first directory. substring : str (default: '') Substring that all files have to contain to be considered. extensions : list (default: None) None or list of allowed file extensions for each path. If provided, the number of extensions must match the number of paths . validity_check : bool (default: None) If True , checks if all dictionary values have the same number of file paths. Prints a warning and returns an empty dictionary if the validity check failed. ignore_invisible : bool (default: True) If True , ignores invisible files (i.e., files starting with a period). rstrip : str (default: '') If provided, strips characters from right side of the file base names after splitting the extension. Useful to trim different filenames to a common stem. E.g,. \"abc_d.txt\" and \"abc_d_.csv\" would share the stem \"abc_d\" if rstrip is set to \"_\". ignore_substring : str (default: None) Ignores files that contain the specified substring. Returns groups : dict Dictionary of files paths. Keys are the file names found in the first directory listed in paths (without file extension). Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/file_io/find_filegroups/","title":"find_filegroups"},{"location":"api_subpackages/mlxtend.file_io/#find_files","text":"find_files(substring, path, recursive=False, check_ext=None, ignore_invisible=True, ignore_substring=None) Find files in a directory based on substring matching. Parameters substring : str Substring of the file to be matched. path : str Path where to look. recursive : bool If true, searches subdirectories recursively. check_ext : str If string (e.g., '.txt'), only returns files that match the specified file extension. ignore_invisible : bool If True , ignores invisible files (i.e., files starting with a period). ignore_substring : str Ignores files that contain the specified substring. Returns results : list List of the matched files. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/file_io/find_files/","title":"find_files"},{"location":"api_subpackages/mlxtend.frequent_patterns/","text":"mlxtend version: 0.23.1 apriori apriori(df, min_support=0.5, use_colnames=False, max_len=None, verbose=0, low_memory=False) Get frequent itemsets from a one-hot DataFrame Parameters df : pandas DataFrame pandas DataFrame the encoded format. Also supports DataFrames with sparse data; for more info, please see (https://pandas.pydata.org/pandas-docs/stable/ user_guide/sparse.html#sparse-data-structures) Please note that the old pandas SparseDataFrame format is no longer supported in mlxtend >= 0.17.2. The allowed values are either 0/1 or True/False. For example, Apple Bananas Beer Chicken Milk Rice 0 True False True True False True 1 True False True False False True 2 True False True False False False 3 True True False False False False 4 False False True True True True 5 False False True False True True 6 False False True False True False 7 True True False False False False min_support : float (default: 0.5) A float between 0 and 1 for minumum support of the itemsets returned. The support is computed as the fraction transactions_where_item(s)_occur / total_transactions . use_colnames : bool (default: False) If True , uses the DataFrames' column names in the returned DataFrame instead of column indices. max_len : int (default: None) Maximum length of the itemsets generated. If None (default) all possible itemsets lengths (under the apriori condition) are evaluated. verbose : int (default: 0) Shows the number of iterations if >= 1 and low_memory is True . If =1 and low_memory is False , shows the number of combinations. low_memory : bool (default: False) If True , uses an iterator to search for combinations above min_support . Note that while low_memory=True should only be used for large dataset if memory resources are limited, because this implementation is approx. 3-6x slower than the default. Returns pandas DataFrame with columns ['support', 'itemsets'] of all itemsets that are >= min_support and < than max_len (if max_len is not None). Each itemset in the 'itemsets' column is of type frozenset , which is a Python built-in type that behaves similarly to sets except that it is immutable (For more info, see https://docs.python.org/3.6/library/stdtypes.html#frozenset). Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/frequent_patterns/apriori/ association_rules association_rules(df, metric='confidence', min_threshold=0.8, support_only=False) Generates a DataFrame of association rules including the metrics 'score', 'confidence', and 'lift' Parameters df : pandas DataFrame pandas DataFrame of frequent itemsets with columns ['support', 'itemsets'] metric : string (default: 'confidence') Metric to evaluate if a rule is of interest. Automatically set to 'support' if support_only=True . Otherwise, supported metrics are 'support', 'confidence', 'lift', 'leverage', 'conviction' and 'zhangs_metric' These metrics are computed as follows: - support(A->C) = support(A+C) [aka 'support'], range: [0, 1] - confidence(A->C) = support(A+C) / support(A), range: [0, 1] - lift(A->C) = confidence(A->C) / support(C), range: [0, inf] - leverage(A->C) = support(A->C) - support(A)*support(C), range: [-1, 1] - conviction = [1 - support(C)] / [1 - confidence(A->C)], range: [0, inf] - zhangs_metric(A->C) = leverage(A->C) / max(support(A->C)*(1-support(A)), support(A)*(support(C)-support(A->C))) range: [-1,1] min_threshold : float (default: 0.8) Minimal threshold for the evaluation metric, via the metric parameter, to decide whether a candidate rule is of interest. support_only : bool (default: False) Only computes the rule support and fills the other metric columns with NaNs. This is useful if: a) the input DataFrame is incomplete, e.g., does not contain support values for all rule antecedents and consequents b) you simply want to speed up the computation because you don't need the other metrics. Returns pandas DataFrame with columns \"antecedents\" and \"consequents\" that store itemsets, plus the scoring metric columns: \"antecedent support\", \"consequent support\", \"support\", \"confidence\", \"lift\", \"leverage\", \"conviction\" of all rules for which metric(rule) >= min_threshold. Each entry in the \"antecedents\" and \"consequents\" columns are of type frozenset , which is a Python built-in type that behaves similarly to sets except that it is immutable (For more info, see https://docs.python.org/3.6/library/stdtypes.html#frozenset). Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/frequent_patterns/association_rules/ fpgrowth fpgrowth(df, min_support=0.5, use_colnames=False, max_len=None, verbose=0) Get frequent itemsets from a one-hot DataFrame Parameters df : pandas DataFrame pandas DataFrame the encoded format. Also supports DataFrames with sparse data; for more info, please see https://pandas.pydata.org/pandas-docs/stable/user_guide/sparse.html#sparse-data-structures. Please note that the old pandas SparseDataFrame format is no longer supported in mlxtend >= 0.17.2. The allowed values are either 0/1 or True/False. For example, Apple Bananas Beer Chicken Milk Rice 0 True False True True False True 1 True False True False False True 2 True False True False False False 3 True True False False False False 4 False False True True True True 5 False False True False True True 6 False False True False True False 7 True True False False False False min_support : float (default: 0.5) A float between 0 and 1 for minimum support of the itemsets returned. The support is computed as the fraction transactions_where_item(s)_occur / total_transactions. use_colnames : bool (default: False) If true, uses the DataFrames' column names in the returned DataFrame instead of column indices. max_len : int (default: None) Maximum length of the itemsets generated. If None (default) all possible itemsets lengths are evaluated. verbose : int (default: 0) Shows the stages of conditional tree generation. Returns pandas DataFrame with columns ['support', 'itemsets'] of all itemsets that are >= min_support and < than max_len (if max_len is not None). Each itemset in the 'itemsets' column is of type frozenset , which is a Python built-in type that behaves similarly to sets except that it is immutable (For more info, see https://docs.python.org/3.6/library/stdtypes.html#frozenset). Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/frequent_patterns/fpgrowth/ fpmax fpmax(df, min_support=0.5, use_colnames=False, max_len=None, verbose=0) Get maximal frequent itemsets from a one-hot DataFrame Parameters df : pandas DataFrame pandas DataFrame the encoded format. Also supports DataFrames with sparse data; for more info, please see (https://pandas.pydata.org/pandas-docs/stable/ user_guide/sparse.html#sparse-data-structures) Please note that the old pandas SparseDataFrame format is no longer supported in mlxtend >= 0.17.2. The allowed values are either 0/1 or True/False. For example, Apple Bananas Beer Chicken Milk Rice 0 True False True True False True 1 True False True False False True 2 True False True False False False 3 True True False False False False 4 False False True True True True 5 False False True False True True 6 False False True False True False 7 True True False False False False min_support : float (default: 0.5) A float between 0 and 1 for minimum support of the itemsets returned. The support is computed as the fraction transactions_where_item(s)_occur / total_transactions. use_colnames : bool (default: False) If true, uses the DataFrames' column names in the returned DataFrame instead of column indices. max_len : int (default: None) Given the set of all maximal itemsets, return those that are less than max_len . If None (default) all possible itemsets lengths are evaluated. verbose : int (default: 0) Shows the stages of conditional tree generation. Returns pandas DataFrame with columns ['support', 'itemsets'] of all maximal itemsets that are >= min_support and < than max_len (if max_len is not None). Each itemset in the 'itemsets' column is of type frozenset , which is a Python built-in type that behaves similarly to sets except that it is immutable (For more info, see https://docs.python.org/3.6/library/stdtypes.html#frozenset). Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/frequent_patterns/fpmax/ hmine hmine(df, min_support=0.5, use_colnames=False, max_len=None, verbose=0) -> pandas.core.frame.DataFrame Get frequent itemsets from a one-hot DataFrame Parameters df : pandas DataFrame pandas DataFrame the encoded format. Also supports DataFrames with sparse data; for more info, please see https://pandas.pydata.org/pandas-docs/stable/user_guide/sparse.html#sparse-data-structures. Please note that the old pandas SparseDataFrame format is no longer supported in mlxtend >= 0.17.2. The allowed values are either 0/1 or True/False. For example, Apple Bananas Beer Chicken Milk Rice 0 True False True True False True 1 True False True False False True 2 True False True False False False 3 True True False False False False 4 False False True True True True 5 False False True False True True 6 False False True False True False 7 True True False False False False min_support : float (default: 0.5) A float between 0 and 1 for minimum support of the itemsets returned. The support is computed as the fraction transactions_where_item(s)_occur / total_transactions. use_colnames : bool (default: False) If true, uses the DataFrames' column names in the returned DataFrame instead of column indices. max_len : int (default: None) Maximum length of the itemsets generated. If None (default) all possible itemsets lengths are evaluated. verbose : int (default: 0) Shows the stages of conditional tree generation. Returns pandas DataFrame with columns ['support', 'itemsets'] of all itemsets that are >= min_support and < than max_len (if max_len is not None). Each itemset in the 'itemsets' column is of type frozenset , which is a Python built-in type that behaves similarly to sets except that it is immutable (For more info, see https://docs.python.org/3.6/library/stdtypes.html#frozenset). Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/frequent_patterns/hmine/","title":"Mlxtend.frequent patterns"},{"location":"api_subpackages/mlxtend.frequent_patterns/#apriori","text":"apriori(df, min_support=0.5, use_colnames=False, max_len=None, verbose=0, low_memory=False) Get frequent itemsets from a one-hot DataFrame Parameters df : pandas DataFrame pandas DataFrame the encoded format. Also supports DataFrames with sparse data; for more info, please see (https://pandas.pydata.org/pandas-docs/stable/ user_guide/sparse.html#sparse-data-structures) Please note that the old pandas SparseDataFrame format is no longer supported in mlxtend >= 0.17.2. The allowed values are either 0/1 or True/False. For example, Apple Bananas Beer Chicken Milk Rice 0 True False True True False True 1 True False True False False True 2 True False True False False False 3 True True False False False False 4 False False True True True True 5 False False True False True True 6 False False True False True False 7 True True False False False False min_support : float (default: 0.5) A float between 0 and 1 for minumum support of the itemsets returned. The support is computed as the fraction transactions_where_item(s)_occur / total_transactions . use_colnames : bool (default: False) If True , uses the DataFrames' column names in the returned DataFrame instead of column indices. max_len : int (default: None) Maximum length of the itemsets generated. If None (default) all possible itemsets lengths (under the apriori condition) are evaluated. verbose : int (default: 0) Shows the number of iterations if >= 1 and low_memory is True . If =1 and low_memory is False , shows the number of combinations. low_memory : bool (default: False) If True , uses an iterator to search for combinations above min_support . Note that while low_memory=True should only be used for large dataset if memory resources are limited, because this implementation is approx. 3-6x slower than the default. Returns pandas DataFrame with columns ['support', 'itemsets'] of all itemsets that are >= min_support and < than max_len (if max_len is not None). Each itemset in the 'itemsets' column is of type frozenset , which is a Python built-in type that behaves similarly to sets except that it is immutable (For more info, see https://docs.python.org/3.6/library/stdtypes.html#frozenset). Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/frequent_patterns/apriori/","title":"apriori"},{"location":"api_subpackages/mlxtend.frequent_patterns/#association_rules","text":"association_rules(df, metric='confidence', min_threshold=0.8, support_only=False) Generates a DataFrame of association rules including the metrics 'score', 'confidence', and 'lift' Parameters df : pandas DataFrame pandas DataFrame of frequent itemsets with columns ['support', 'itemsets'] metric : string (default: 'confidence') Metric to evaluate if a rule is of interest. Automatically set to 'support' if support_only=True . Otherwise, supported metrics are 'support', 'confidence', 'lift', 'leverage', 'conviction' and 'zhangs_metric' These metrics are computed as follows: - support(A->C) = support(A+C) [aka 'support'], range: [0, 1] - confidence(A->C) = support(A+C) / support(A), range: [0, 1] - lift(A->C) = confidence(A->C) / support(C), range: [0, inf] - leverage(A->C) = support(A->C) - support(A)*support(C), range: [-1, 1] - conviction = [1 - support(C)] / [1 - confidence(A->C)], range: [0, inf] - zhangs_metric(A->C) = leverage(A->C) / max(support(A->C)*(1-support(A)), support(A)*(support(C)-support(A->C))) range: [-1,1] min_threshold : float (default: 0.8) Minimal threshold for the evaluation metric, via the metric parameter, to decide whether a candidate rule is of interest. support_only : bool (default: False) Only computes the rule support and fills the other metric columns with NaNs. This is useful if: a) the input DataFrame is incomplete, e.g., does not contain support values for all rule antecedents and consequents b) you simply want to speed up the computation because you don't need the other metrics. Returns pandas DataFrame with columns \"antecedents\" and \"consequents\" that store itemsets, plus the scoring metric columns: \"antecedent support\", \"consequent support\", \"support\", \"confidence\", \"lift\", \"leverage\", \"conviction\" of all rules for which metric(rule) >= min_threshold. Each entry in the \"antecedents\" and \"consequents\" columns are of type frozenset , which is a Python built-in type that behaves similarly to sets except that it is immutable (For more info, see https://docs.python.org/3.6/library/stdtypes.html#frozenset). Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/frequent_patterns/association_rules/","title":"association_rules"},{"location":"api_subpackages/mlxtend.frequent_patterns/#fpgrowth","text":"fpgrowth(df, min_support=0.5, use_colnames=False, max_len=None, verbose=0) Get frequent itemsets from a one-hot DataFrame Parameters df : pandas DataFrame pandas DataFrame the encoded format. Also supports DataFrames with sparse data; for more info, please see https://pandas.pydata.org/pandas-docs/stable/user_guide/sparse.html#sparse-data-structures. Please note that the old pandas SparseDataFrame format is no longer supported in mlxtend >= 0.17.2. The allowed values are either 0/1 or True/False. For example, Apple Bananas Beer Chicken Milk Rice 0 True False True True False True 1 True False True False False True 2 True False True False False False 3 True True False False False False 4 False False True True True True 5 False False True False True True 6 False False True False True False 7 True True False False False False min_support : float (default: 0.5) A float between 0 and 1 for minimum support of the itemsets returned. The support is computed as the fraction transactions_where_item(s)_occur / total_transactions. use_colnames : bool (default: False) If true, uses the DataFrames' column names in the returned DataFrame instead of column indices. max_len : int (default: None) Maximum length of the itemsets generated. If None (default) all possible itemsets lengths are evaluated. verbose : int (default: 0) Shows the stages of conditional tree generation. Returns pandas DataFrame with columns ['support', 'itemsets'] of all itemsets that are >= min_support and < than max_len (if max_len is not None). Each itemset in the 'itemsets' column is of type frozenset , which is a Python built-in type that behaves similarly to sets except that it is immutable (For more info, see https://docs.python.org/3.6/library/stdtypes.html#frozenset). Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/frequent_patterns/fpgrowth/","title":"fpgrowth"},{"location":"api_subpackages/mlxtend.frequent_patterns/#fpmax","text":"fpmax(df, min_support=0.5, use_colnames=False, max_len=None, verbose=0) Get maximal frequent itemsets from a one-hot DataFrame Parameters df : pandas DataFrame pandas DataFrame the encoded format. Also supports DataFrames with sparse data; for more info, please see (https://pandas.pydata.org/pandas-docs/stable/ user_guide/sparse.html#sparse-data-structures) Please note that the old pandas SparseDataFrame format is no longer supported in mlxtend >= 0.17.2. The allowed values are either 0/1 or True/False. For example, Apple Bananas Beer Chicken Milk Rice 0 True False True True False True 1 True False True False False True 2 True False True False False False 3 True True False False False False 4 False False True True True True 5 False False True False True True 6 False False True False True False 7 True True False False False False min_support : float (default: 0.5) A float between 0 and 1 for minimum support of the itemsets returned. The support is computed as the fraction transactions_where_item(s)_occur / total_transactions. use_colnames : bool (default: False) If true, uses the DataFrames' column names in the returned DataFrame instead of column indices. max_len : int (default: None) Given the set of all maximal itemsets, return those that are less than max_len . If None (default) all possible itemsets lengths are evaluated. verbose : int (default: 0) Shows the stages of conditional tree generation. Returns pandas DataFrame with columns ['support', 'itemsets'] of all maximal itemsets that are >= min_support and < than max_len (if max_len is not None). Each itemset in the 'itemsets' column is of type frozenset , which is a Python built-in type that behaves similarly to sets except that it is immutable (For more info, see https://docs.python.org/3.6/library/stdtypes.html#frozenset). Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/frequent_patterns/fpmax/","title":"fpmax"},{"location":"api_subpackages/mlxtend.frequent_patterns/#hmine","text":"hmine(df, min_support=0.5, use_colnames=False, max_len=None, verbose=0) -> pandas.core.frame.DataFrame Get frequent itemsets from a one-hot DataFrame Parameters df : pandas DataFrame pandas DataFrame the encoded format. Also supports DataFrames with sparse data; for more info, please see https://pandas.pydata.org/pandas-docs/stable/user_guide/sparse.html#sparse-data-structures. Please note that the old pandas SparseDataFrame format is no longer supported in mlxtend >= 0.17.2. The allowed values are either 0/1 or True/False. For example, Apple Bananas Beer Chicken Milk Rice 0 True False True True False True 1 True False True False False True 2 True False True False False False 3 True True False False False False 4 False False True True True True 5 False False True False True True 6 False False True False True False 7 True True False False False False min_support : float (default: 0.5) A float between 0 and 1 for minimum support of the itemsets returned. The support is computed as the fraction transactions_where_item(s)_occur / total_transactions. use_colnames : bool (default: False) If true, uses the DataFrames' column names in the returned DataFrame instead of column indices. max_len : int (default: None) Maximum length of the itemsets generated. If None (default) all possible itemsets lengths are evaluated. verbose : int (default: 0) Shows the stages of conditional tree generation. Returns pandas DataFrame with columns ['support', 'itemsets'] of all itemsets that are >= min_support and < than max_len (if max_len is not None). Each itemset in the 'itemsets' column is of type frozenset , which is a Python built-in type that behaves similarly to sets except that it is immutable (For more info, see https://docs.python.org/3.6/library/stdtypes.html#frozenset). Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/frequent_patterns/hmine/","title":"hmine"},{"location":"api_subpackages/mlxtend.image/","text":"mlxtend version: 0.23.1 EyepadAlign EyepadAlign(verbose=0) Class to align/transform face images to facial landmarks, based on eye alignment. 1. A scaling factor is computed based on distance between the left and right eye, such that the transformed face image will have the same eye distance as a reference face image. 2. A transformation is performed based on the eyes' center point. to align the face based on the reference eye location. 3. Finally, the transformed image is padded with zeros to match the desired final image size. Parameters verbose : int (default=0) Verbose level to display the progress bar and log messages. Setting verbose=1 will print a progress bar upon calling fit_directory . Attributes target_landmarks_ : target landmarks to transform new face images to. Depending on the chosen fit parameters, it can be either (1) assigned to pre-fit shapes, (2) computed from a single face image (3) computed as the mean of face landmarks from all face images in a file directory of face images. eye_distance_ : the distance between left and right eyes in the target landmarks. target_height_ : the height of the transformed output image. target_width_ : the width of the transformed output image. Returns self : object Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/image/eyepad_align/ Methods fit_directory(target_img_dir, target_height, target_width, file_extension='.jpg', pre_check=True) Calculates the average landmarks for all face images in a directory which will then be set as the target landmark set. Arguments target_img_dir : str Directory containing the images target_height : int Expected image height of the images in the directory target_width : int Expected image width of the images in the directory file_extension str (default='.jpg'): File extension of the image files. pre_check Bool (default=True): Checks that each image has the dimensions specificed via target_height and target_width on the whole directory first to identify potential issues that are recommended to be fixed before proceeding. Raises a warning for each image if dimensions differ from the ones specified and expected. Returns self : object fit_image(target_image) Derives facial landmarks from a target image. Arguments target_image : uint8 numpy.array, shape=[width, height, channels] NumPy array representation of the image data. Returns self : object fit_values(target_landmarks, target_width, target_height) Used for determining the eye location from pre-defined landmark arrays, eliminating the need for re-computing the average landmarks on a target image or image directory. Arguments target_landmarks : np.array, shape=(height, width) NumPy array containing the locations of the facial landmarks as determined by mlxtend.image.extract_face_landmarks target_height : int image height target_width : int image width Returns self : object transform(img) transforms a single face image (img) to the target landmarks based on the location of the eyes by scaling, translation and cropping (if needed): (1) Scaling the image so that the distance of the two eyes in the given image (img) matches the distance of the two eyes in the target landmarks. (2) Translation is performed based on the middle point between the two eyes. Arguments img : np.array, shape=(height, width, channels) Input image to be transformed. Returns self : object extract_face_landmarks extract_face_landmarks(img, return_dtype= ) Function to extract face landmarks. Note that this function requires an installation of the Python version of the library \"dlib\": https://dlib.net Parameters img : array, shape = [h, w, ?] Numpy array of a face image or imageio.core.util.Array. E.g., img = imageio.core.util.Array(ary) Supported shapes are - 3D tensors with 1 or more color channels, for example, RGB: [h, w, 3] - 2D tensors without color channel, for example, Grayscale: [h, w] return_dtype: the return data-type of the array, default: np.int32. Returns landmarks : numpy.ndarray, shape = [68, 2] A numpy array, where each row contains a landmark/point x-y coordinates. Return None if no face is detected by Dlib. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/image/extract_face_landmarks/","title":"Mlxtend.image"},{"location":"api_subpackages/mlxtend.image/#eyepadalign","text":"EyepadAlign(verbose=0) Class to align/transform face images to facial landmarks, based on eye alignment. 1. A scaling factor is computed based on distance between the left and right eye, such that the transformed face image will have the same eye distance as a reference face image. 2. A transformation is performed based on the eyes' center point. to align the face based on the reference eye location. 3. Finally, the transformed image is padded with zeros to match the desired final image size. Parameters verbose : int (default=0) Verbose level to display the progress bar and log messages. Setting verbose=1 will print a progress bar upon calling fit_directory . Attributes target_landmarks_ : target landmarks to transform new face images to. Depending on the chosen fit parameters, it can be either (1) assigned to pre-fit shapes, (2) computed from a single face image (3) computed as the mean of face landmarks from all face images in a file directory of face images. eye_distance_ : the distance between left and right eyes in the target landmarks. target_height_ : the height of the transformed output image. target_width_ : the width of the transformed output image. Returns self : object Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/image/eyepad_align/","title":"EyepadAlign"},{"location":"api_subpackages/mlxtend.image/#methods","text":"fit_directory(target_img_dir, target_height, target_width, file_extension='.jpg', pre_check=True) Calculates the average landmarks for all face images in a directory which will then be set as the target landmark set. Arguments target_img_dir : str Directory containing the images target_height : int Expected image height of the images in the directory target_width : int Expected image width of the images in the directory file_extension str (default='.jpg'): File extension of the image files. pre_check Bool (default=True): Checks that each image has the dimensions specificed via target_height and target_width on the whole directory first to identify potential issues that are recommended to be fixed before proceeding. Raises a warning for each image if dimensions differ from the ones specified and expected. Returns self : object fit_image(target_image) Derives facial landmarks from a target image. Arguments target_image : uint8 numpy.array, shape=[width, height, channels] NumPy array representation of the image data. Returns self : object fit_values(target_landmarks, target_width, target_height) Used for determining the eye location from pre-defined landmark arrays, eliminating the need for re-computing the average landmarks on a target image or image directory. Arguments target_landmarks : np.array, shape=(height, width) NumPy array containing the locations of the facial landmarks as determined by mlxtend.image.extract_face_landmarks target_height : int image height target_width : int image width Returns self : object transform(img) transforms a single face image (img) to the target landmarks based on the location of the eyes by scaling, translation and cropping (if needed): (1) Scaling the image so that the distance of the two eyes in the given image (img) matches the distance of the two eyes in the target landmarks. (2) Translation is performed based on the middle point between the two eyes. Arguments img : np.array, shape=(height, width, channels) Input image to be transformed. Returns self : object","title":"Methods"},{"location":"api_subpackages/mlxtend.image/#extract_face_landmarks","text":"extract_face_landmarks(img, return_dtype= ) Function to extract face landmarks. Note that this function requires an installation of the Python version of the library \"dlib\": https://dlib.net Parameters img : array, shape = [h, w, ?] Numpy array of a face image or imageio.core.util.Array. E.g., img = imageio.core.util.Array(ary) Supported shapes are - 3D tensors with 1 or more color channels, for example, RGB: [h, w, 3] - 2D tensors without color channel, for example, Grayscale: [h, w] return_dtype: the return data-type of the array, default: np.int32. Returns landmarks : numpy.ndarray, shape = [68, 2] A numpy array, where each row contains a landmark/point x-y coordinates. Return None if no face is detected by Dlib. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/image/extract_face_landmarks/","title":"extract_face_landmarks"},{"location":"api_subpackages/mlxtend.math/","text":"mlxtend version: 0.23.1 factorial factorial(n) None num_combinations num_combinations(n, k, with_replacement=False) Function to calculate the number of possible combinations. Parameters n : int Total number of items. k : int Number of elements of the target itemset. with_replacement : bool (default: False) Allows repeated elements if True. Returns comb : int Number of possible combinations. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/math/num_combinations/ num_permutations num_permutations(n, k, with_replacement=False) Function to calculate the number of possible permutations. Parameters n : int Total number of items. k : int Number of elements of the target itemset. with_replacement : bool Allows repeated elements if True. Returns permut : int Number of possible permutations. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/math/num_permutations/ vectorspace_dimensionality vectorspace_dimensionality(ary) Computes the hyper-volume spanned by a vector set Parameters ary : array-like, shape=[num_vectors, num_vectors] An orthogonal set of vectors (arranged as columns in a matrix) Returns dimensions : int An integer indicating the \"dimensionality\" hyper-volume spanned by the vector set Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/math/vectorspace_dimensionality/ vectorspace_orthonormalization vectorspace_orthonormalization(ary, eps=1e-13) Transforms a set of column vectors to a orthonormal basis. Given a set of orthogonal vectors, this functions converts such column vectors, arranged in a matrix, into orthonormal basis vectors. Parameters ary : array-like, shape=[num_vectors, num_vectors] An orthogonal set of vectors (arranged as columns in a matrix) eps : float (default: 1e-13) A small tolerance value to determine whether the vector norm is zero or not. Returns arr : array-like, shape=[num_vectors, num_vectors] An orthonormal set of vectors (arranged as columns) Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/math/vectorspace_orthonormalization/","title":"Mlxtend.math"},{"location":"api_subpackages/mlxtend.math/#factorial","text":"factorial(n) None","title":"factorial"},{"location":"api_subpackages/mlxtend.math/#num_combinations","text":"num_combinations(n, k, with_replacement=False) Function to calculate the number of possible combinations. Parameters n : int Total number of items. k : int Number of elements of the target itemset. with_replacement : bool (default: False) Allows repeated elements if True. Returns comb : int Number of possible combinations. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/math/num_combinations/","title":"num_combinations"},{"location":"api_subpackages/mlxtend.math/#num_permutations","text":"num_permutations(n, k, with_replacement=False) Function to calculate the number of possible permutations. Parameters n : int Total number of items. k : int Number of elements of the target itemset. with_replacement : bool Allows repeated elements if True. Returns permut : int Number of possible permutations. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/math/num_permutations/","title":"num_permutations"},{"location":"api_subpackages/mlxtend.math/#vectorspace_dimensionality","text":"vectorspace_dimensionality(ary) Computes the hyper-volume spanned by a vector set Parameters ary : array-like, shape=[num_vectors, num_vectors] An orthogonal set of vectors (arranged as columns in a matrix) Returns dimensions : int An integer indicating the \"dimensionality\" hyper-volume spanned by the vector set Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/math/vectorspace_dimensionality/","title":"vectorspace_dimensionality"},{"location":"api_subpackages/mlxtend.math/#vectorspace_orthonormalization","text":"vectorspace_orthonormalization(ary, eps=1e-13) Transforms a set of column vectors to a orthonormal basis. Given a set of orthogonal vectors, this functions converts such column vectors, arranged in a matrix, into orthonormal basis vectors. Parameters ary : array-like, shape=[num_vectors, num_vectors] An orthogonal set of vectors (arranged as columns in a matrix) eps : float (default: 1e-13) A small tolerance value to determine whether the vector norm is zero or not. Returns arr : array-like, shape=[num_vectors, num_vectors] An orthonormal set of vectors (arranged as columns) Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/math/vectorspace_orthonormalization/","title":"vectorspace_orthonormalization"},{"location":"api_subpackages/mlxtend.plotting/","text":"mlxtend version: 0.23.1 category_scatter category_scatter(x, y, label_col, data, markers='sxo^v', colors=('blue', 'green', 'red', 'purple', 'gray', 'cyan'), alpha=0.7, markersize=20.0, legend_loc='best') Scatter plot to plot categories in different colors/markerstyles. Parameters x : str or int DataFrame column name of the x-axis values or integer for the numpy ndarray column index. y : str DataFrame column name of the y-axis values or integer for the numpy ndarray column index data : Pandas DataFrame object or NumPy ndarray. markers : str Markers that are cycled through the label category. colors : tuple Colors that are cycled through the label category. alpha : float (default: 0.7) Parameter to control the transparency. markersize : float (default` : 20.0) Parameter to control the marker size. legend_loc : str (default: 'best') Location of the plot legend {best, upper left, upper right, lower left, lower right} No legend if legend_loc=False Returns fig : matplotlig.pyplot figure object Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/plotting/category_scatter/ checkerboard_plot checkerboard_plot(ary, cell_colors=('white', 'black'), font_colors=('black', 'white'), fmt='%.1f', figsize=None, row_labels=None, col_labels=None, fontsize=None) Plot a checkerboard table / heatmap via matplotlib. Parameters ary : array-like, shape = [n, m] A 2D Nnumpy array. cell_colors : tuple or list (default: ('white', 'black')) Tuple or list containing the two colors of the checkerboard pattern. font_colors : tuple or list (default: ('black', 'white')) Font colors corresponding to the cell colors. figsize : tuple (default: (2.5, 2.5)) Height and width of the figure fmt : str (default: '%.1f') Python string formatter for cell values. The default '%.1f' results in floats with 1 digit after the decimal point. Use '%d' to show numbers as integers. row_labels : list (default: None) List of the row labels. Uses the array row indices 0 to n by default. col_labels : list (default: None) List of the column labels. Uses the array column indices 0 to m by default. fontsize : int (default: None) Specifies the font size of the checkerboard table. Uses matplotlib's default if None. Returns fig : matplotlib Figure object. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/plotting/checkerboard_plot/ ecdf ecdf(x, y_label='ECDF', x_label=None, ax=None, percentile=None, ecdf_color=None, ecdf_marker='o', percentile_color='black', percentile_linestyle='--') Plots an Empirical Cumulative Distribution Function Parameters x : array or list, shape=[n_samples,] Array-like object containing the feature values y_label : str (default='ECDF') Text label for the y-axis x_label : str (default=None) Text label for the x-axis ax : matplotlib.axes.Axes (default: None) An existing matplotlib Axes. Creates one if ax=None percentile : float (default=None) Float between 0 and 1 for plotting a percentile threshold line ecdf_color : matplotlib color (default=None) Color for the ECDF plot; uses matplotlib defaults if None ecdf_marker : matplotlib marker (default='o') Marker style for the ECDF plot percentile_color : matplotlib color (default='black') Color for the percentile threshold if percentile is not None percentile_linestyle : matplotlib linestyle (default='--') Line style for the percentile threshold if percentile is not None Returns ax : matplotlib.axes.Axes object percentile_threshold : float Feature threshold at the percentile or None if percentile=None percentile_count : Number of if percentile is not None Number of samples that have a feature less or equal than the feature threshold at a percentile threshold or None if percentile=None Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/plotting/ecdf/ enrichment_plot enrichment_plot(df, colors='bgrkcy', markers=' ', linestyles='-', alpha=0.5, lw=2, where='post', grid=True, count_label='Count', xlim='auto', ylim='auto', invert_axes=False, legend_loc='best', ax=None) Plot stacked barplots Parameters df : pandas.DataFrame A pandas DataFrame where columns represent the different categories. colors: str (default: 'bgrcky') The colors of the bars. markers : str (default: ' ') Matplotlib markerstyles, e.g, 'sov' for square,circle, and triangle markers. linestyles : str (default: '-') Matplotlib linestyles, e.g., '-,--' to cycle normal and dashed lines. Note that the different linestyles need to be separated by commas. alpha : float (default: 0.5) Transparency level from 0.0 to 1.0. lw : int or float (default: 2) Linewidth parameter. where : {'post', 'pre', 'mid'} (default: 'post') Starting location of the steps. grid : bool (default: True ) Plots a grid if True. count_label : str (default: 'Count') Label for the \"Count\"-axis. xlim : 'auto' or array-like [min, max] (default: 'auto') Min and maximum position of the x-axis range. ylim : 'auto' or array-like [min, max] (default: 'auto') Min and maximum position of the y-axis range. invert_axes : bool (default: False) Plots count on the x-axis if True. legend_loc : str (default: 'best') Location of the plot legend {best, upper left, upper right, lower left, lower right} No legend if legend_loc=False ax : matplotlib axis, optional (default: None) Use this axis for plotting or make a new one otherwise Returns ax : matplotlib axis Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/plotting/enrichment_plot/ heatmap heatmap(matrix, hide_spines=False, hide_ticks=False, figsize=None, cmap=None, colorbar=True, row_names=None, column_names=None, column_name_rotation=45, cell_values=True, cell_fmt='.2f', cell_font_size=None, text_color_threshold=None) Plot a heatmap via matplotlib. Parameters conf_mat : array-like, shape = [n_rows, n_columns] And arbitrary 2D array. hide_spines : bool (default: False) Hides axis spines if True. hide_ticks : bool (default: False) Hides axis ticks if True figsize : tuple (default: (2.5, 2.5)) Height and width of the figure cmap : matplotlib colormap (default: None ) Uses matplotlib.pyplot.cm.viridis if None colorbar : bool (default: True) Shows a colorbar if True row_names : array-like, shape = [n_rows] (default: None) List of row names to be used as y-axis tick labels. column_names : array-like, shape = [n_columns] (default: None) List of column names to be used as x-axis tick labels. column_name_rotation : int (default: 45) Number of degrees for rotating column x-tick labels. cell_values : bool (default: True) Plots cell values if True. cell_fmt : string (default: '.2f') Format specification for cell values (if cell_values=True ) cell_font_size : int (default: None) Font size for cell values (if cell_values=True ) text_color_threshold : float (default: None) Threshold for the black/white text threshold of the text annotation. Default (None) tried to infer a good threshold automatically using np.max(normed_matrix) / 2 . Returns fig, ax : matplotlib.pyplot subplot objects Figure and axis elements of the subplot. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/plotting/heatmap/ plot_confusion_matrix plot_confusion_matrix(conf_mat, hide_spines=False, hide_ticks=False, figsize=None, cmap=None, colorbar=False, show_absolute=True, show_normed=False, norm_colormap=None, class_names=None, figure=None, axis=None, fontcolor_threshold=0.5) Plot a confusion matrix via matplotlib. Parameters conf_mat : array-like, shape = [n_classes, n_classes] Confusion matrix from evaluate.confusion matrix. hide_spines : bool (default: False) Hides axis spines if True. hide_ticks : bool (default: False) Hides axis ticks if True figsize : tuple (default: (2.5, 2.5)) Height and width of the figure cmap : matplotlib colormap (default: None ) Uses matplotlib.pyplot.cm.Blues if None colorbar : bool (default: False) Shows a colorbar if True show_absolute : bool (default: True) Shows absolute confusion matrix coefficients if True. At least one of show_absolute or show_normed must be True. show_normed : bool (default: False) Shows normed confusion matrix coefficients if True. The normed confusion matrix coefficients give the proportion of training examples per class that are assigned the correct label. At least one of show_absolute or show_normed must be True. norm_colormap : bool (default: False) Matplotlib color normalization object to normalize the color scale, e.g., matplotlib.colors.LogNorm() . class_names : array-like, shape = [n_classes] (default: None) List of class names. If not None , ticks will be set to these values. figure : None or Matplotlib figure (default: None) If None will create a new figure. axis : None or Matplotlib figure axis (default: None) If None will create a new axis. fontcolor_threshold : Float (default: 0.5) Sets a threshold for choosing black and white font colors for the cells. By default all values larger than 0.5 times the maximum cell value are converted to white, and everything equal or smaller than 0.5 times the maximum cell value are converted to black. Returns fig, ax : matplotlib.pyplot subplot objects Figure and axis elements of the subplot. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/plotting/plot_confusion_matrix/ plot_decision_regions plot_decision_regions(X, y, clf, feature_index=None, filler_feature_values=None, filler_feature_ranges=None, ax=None, X_highlight=None, zoom_factor=1.0, legend=1, hide_spines=True, markers='s^oxv<>', colors='#1f77b4,#ff7f0e,#3ca02c,#d62728,#9467bd,#8c564b,#e377c2,#7f7f7f,#bcbd22,#17becf', scatter_kwargs=None, contourf_kwargs=None, contour_kwargs=None, scatter_highlight_kwargs=None, n_jobs=None) Plot decision regions of a classifier. Please note that this functions assumes that class labels are labeled consecutively, e.g,. 0, 1, 2, 3, 4, and 5. If you have class labels with integer labels > 4, you may want to provide additional colors and/or markers as `colors` and `markers` arguments. See https://matplotlib.org/examples/color/named_colors.html for more information. Parameters X : array-like, shape = [n_samples, n_features] Feature Matrix. y : array-like, shape = [n_samples] True class labels. clf : Classifier object. Must have a .predict method. feature_index : array-like (default: (0,) for 1D, (0, 1) otherwise) Feature indices to use for plotting. The first index in feature_index will be on the x-axis, the second index will be on the y-axis. filler_feature_values : dict (default: None) Only needed for number features > 2. Dictionary of feature index-value pairs for the features not being plotted. filler_feature_ranges : dict (default: None) Only needed for number features > 2. Dictionary of feature index-value pairs for the features not being plotted. Will use the ranges provided to select training samples for plotting. ax : matplotlib.axes.Axes (default: None) An existing matplotlib Axes. Creates one if ax=None. X_highlight : array-like, shape = [n_samples, n_features] (default: None) An array with data points that are used to highlight samples in X . zoom_factor : float (default: 1.0) Controls the scale of the x- and y-axis of the decision plot. hide_spines : bool (default: True) Hide axis spines if True. legend : int (default: 1) Integer to specify the legend location. No legend if legend is 0. markers : str (default: 's^oxv<>') Scatterplot markers. colors : str (default: 'red,blue,limegreen,gray,cyan') Comma separated list of colors. scatter_kwargs : dict (default: None) Keyword arguments for underlying matplotlib scatter function. contourf_kwargs : dict (default: None) Keyword arguments for underlying matplotlib contourf function. contour_kwargs : dict (default: None) Keyword arguments for underlying matplotlib contour function (which draws the lines between decision regions). scatter_highlight_kwargs : dict (default: None) Keyword arguments for underlying matplotlib scatter function. n_jobs : int or None, optional (default=None) The number of CPUs to use to do the computation using Python's multiprocessing library. None means 1. -1 means using all processors. New in v0.22.0. Returns ax : matplotlib.axes.Axes object Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/plotting/plot_decision_regions/ plot_learning_curves plot_learning_curves(X_train, y_train, X_test, y_test, clf, train_marker='o', test_marker='^', scoring='misclassification error', suppress_plot=False, print_model=True, title_fontsize=12, style='default', legend_loc='best') Plots learning curves of a classifier. Parameters X_train : array-like, shape = [n_samples, n_features] Feature matrix of the training dataset. y_train : array-like, shape = [n_samples] True class labels of the training dataset. X_test : array-like, shape = [n_samples, n_features] Feature matrix of the test dataset. y_test : array-like, shape = [n_samples] True class labels of the test dataset. clf : Classifier object. Must have a .predict .fit method. train_marker : str (default: 'o') Marker for the training set line plot. test_marker : str (default: '^') Marker for the test set line plot. scoring : str (default: 'misclassification error') If not 'misclassification error', accepts the following metrics (from scikit-learn): {'accuracy', 'average_precision', 'f1_micro', 'f1_macro', 'f1_weighted', 'f1_samples', 'log_loss', 'precision', 'recall', 'roc_auc', 'adjusted_rand_score', 'mean_absolute_error', 'mean_squared_error', 'median_absolute_error', 'r2'} suppress_plot=False : bool (default: False) Suppress matplotlib plots if True. Recommended for testing purposes. print_model : bool (default: True) Print model parameters in plot title if True. title_fontsize : int (default: 12) Determines the size of the plot title font. style : str (default: 'default') Matplotlib style. For more styles, please see https://matplotlib.org/stable/gallery/style_sheets/style_sheets_reference.html legend_loc : str (default: 'best') Where to place the plot legend: {'best', 'upper left', 'upper right', 'lower left', 'lower right'} Returns errors : (training_error, test_error): tuple of lists Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/plotting/plot_learning_curves/ plot_linear_regression plot_linear_regression(X, y, model=LinearRegression(), corr_func='pearsonr', scattercolor='blue', fit_style='k--', legend=True, xlim='auto') Plot a linear regression line fit. Parameters X : numpy array, shape = [n_samples,] Samples. y : numpy array, shape (n_samples,) Target values model: object (default: sklearn.linear_model.LinearRegression) Estimator object for regression. Must implement a .fit() and .predict() method. corr_func: str or function (default: 'pearsonr') Uses pearsonr from scipy.stats if corr_func='pearsonr'. to compute the regression slope. If not 'pearsonr', the corr_func , the corr_func parameter expects a function of the form func( , ) as inputs, which is expected to return a tuple (<correlation_coefficient>, <some_unused_value>) . scattercolor: string (default: blue) Color of scatter plot points. fit_style: string (default: k--) Style for the line fit. legend: bool (default: True) Plots legend with corr_coeff coef., fit coef., and intercept values. xlim: array-like (x_min, x_max) or 'auto' (default: 'auto') X-axis limits for the linear line fit. Returns regression_fit : tuple intercept, slope, corr_coeff (float, float, float) Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/plotting/plot_linear_regression/ plot_pca_correlation_graph plot_pca_correlation_graph(X, variables_names, dimensions=(1, 2), figure_axis_size=6, X_pca=None, explained_variance=None) Compute the PCA for X and plots the Correlation graph Parameters X : 2d array like. The columns represent the different variables and the rows are the samples of thos variables variables_names : array like Name of the columns (the variables) of X dimensions: tuple with two elements. dimensions to be plotted (x,y) figure_axis_size : size of the final frame. The figure created is a square with length and width equal to figure_axis_size. X_pca : np.ndarray, shape = [n_samples, n_components]. Optional. X_pca is the matrix of the transformed components from X. If not provided, the function computes PCA automatically using mlxtend.feature_extraction.PrincipalComponentAnalysis Expected n_componentes >= max(dimensions) explained_variance : 1 dimension np.ndarray, length = n_components Optional. explained_variance are the eigenvalues from the diagonalized covariance matrix on the PCA transformatiopn. If not provided, the function computes PCA independently Expected n_componentes == X.shape[1] Returns matplotlib_figure, correlation_matrix Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/plotting/plot_pca_correlation_graph/ plot_sequential_feature_selection plot_sequential_feature_selection(metric_dict, figsize=None, kind='std_dev', color='blue', bcolor='steelblue', marker='o', alpha=0.2, ylabel='Performance', confidence_interval=0.95) Plot feature selection results. Parameters metric_dict : mlxtend.SequentialFeatureSelector.get_metric_dict() object figsize : tuple (default: None) Height and width of the figure kind : str (default: \"std_dev\") The kind of error bar or confidence interval in {'std_dev', 'std_err', 'ci', None}. color : str (default: \"blue\") Color of the lineplot (accepts any matplotlib color name) bcolor : str (default: \"steelblue\"). Color of the error bars / confidence intervals (accepts any matplotlib color name). marker : str (default: \"o\") Marker of the line plot (accepts any matplotlib marker name). alpha : float in [0, 1] (default: 0.2) Transparency of the error bars / confidence intervals. ylabel : str (default: \"Performance\") Y-axis label. confidence_interval : float (default: 0.95) Confidence level if kind='ci' . Returns fig : matplotlib.pyplot.figure() object Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/plotting/plot_sequential_feature_selection/ remove_borders remove_borders(axes, left=False, bottom=False, right=True, top=True) Remove chart junk from matplotlib plots. Parameters axes : iterable An iterable containing plt.gca() or plt.subplot() objects, e.g. [plt.gca()]. left : bool (default: False ) Hide left axis spine if True. bottom : bool (default: False ) Hide bottom axis spine if True. right : bool (default: True ) Hide right axis spine if True. top : bool (default: True ) Hide top axis spine if True. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/plotting/remove_chartjunk/ scatter_hist scatter_hist(x, y, xlabel=None, ylabel=None, figsize=(5, 5)) Scatter plot and individual feature histograms along axes. Parameters x : 1D array-like or Pandas Series X-axis values. y : 1D array-like or Pandas Series Y-axis values. xlabel : str (default: None ) Label for the X-axis values. If x is a pandas Series, and xlabel is None , the label is inferred automatically. ylabel : str (default: None ) Label for the X-axis values. If y is a pandas Series, and ylabel is None , the label is inferred automatically. figsize : tuple (default: (5, 5) ) Matplotlib figure size. Returns plot : Matplotlib Figure object scatterplotmatrix scatterplotmatrix(X, fig_axes=None, names=None, figsize=(8, 8), alpha=1.0, kwargs) Lower triangular of a scatterplot matrix Parameters X : array-like, shape={num_examples, num_features} Design matrix containing data instances (examples) with multiple exploratory variables (features). fix_axes : tuple (default: None) A (fig, axes) tuple, where fig is an figure object and axes is an axes object created via matplotlib, for example, by calling the pyplot subplot function fig, axes = plt.subplots(...) names : list (default: None) A list of string names, which should have the same number of elements as there are features (columns) in X . figsize : tuple (default: (8, 8)) Height and width of the subplot grid. Ignored if fig_axes is not None . alpha : float (default: 1.0) Transparency for both the scatter plots and the histograms along the diagonal. **kwargs : kwargs Keyword arguments for the scatterplots. Returns fix_axes : tuple A (fig, axes) tuple, where fig is an figure object and axes is an axes object created via matplotlib, for example, by calling the pyplot subplot function fig, axes = plt.subplots(...) Examples For more usage examples, please see https://rasbt.github.io/mlxtend/user_guide/plotting/scatterplotmatrix/ stacked_barplot stacked_barplot(df, bar_width='auto', colors='bgrcky', labels='index', rotation=90, legend_loc='best') Function to plot stacked barplots Parameters df : pandas.DataFrame A pandas DataFrame where the index denotes the x-axis labels, and the columns contain the different measurements for each row. bar_width: 'auto' or float (default: 'auto') Parameter to set the widths of the bars. if 'auto', the width is automatically determined by the number of columns in the dataset. colors: str (default: 'bgrcky') The colors of the bars. labels: 'index' or iterable (default: 'index') If 'index', the DataFrame index will be used as x-tick labels. rotation: int (default: 90) Parameter to rotate the x-axis labels. legend_loc : str (default: 'best') Location of the plot legend {best, upper left, upper right, lower left, lower right} No legend if legend_loc=False Returns fig : matplotlib.pyplot figure object Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/plotting/stacked_barplot/","title":"Mlxtend.plotting"},{"location":"api_subpackages/mlxtend.plotting/#category_scatter","text":"category_scatter(x, y, label_col, data, markers='sxo^v', colors=('blue', 'green', 'red', 'purple', 'gray', 'cyan'), alpha=0.7, markersize=20.0, legend_loc='best') Scatter plot to plot categories in different colors/markerstyles. Parameters x : str or int DataFrame column name of the x-axis values or integer for the numpy ndarray column index. y : str DataFrame column name of the y-axis values or integer for the numpy ndarray column index data : Pandas DataFrame object or NumPy ndarray. markers : str Markers that are cycled through the label category. colors : tuple Colors that are cycled through the label category. alpha : float (default: 0.7) Parameter to control the transparency. markersize : float (default` : 20.0) Parameter to control the marker size. legend_loc : str (default: 'best') Location of the plot legend {best, upper left, upper right, lower left, lower right} No legend if legend_loc=False Returns fig : matplotlig.pyplot figure object Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/plotting/category_scatter/","title":"category_scatter"},{"location":"api_subpackages/mlxtend.plotting/#checkerboard_plot","text":"checkerboard_plot(ary, cell_colors=('white', 'black'), font_colors=('black', 'white'), fmt='%.1f', figsize=None, row_labels=None, col_labels=None, fontsize=None) Plot a checkerboard table / heatmap via matplotlib. Parameters ary : array-like, shape = [n, m] A 2D Nnumpy array. cell_colors : tuple or list (default: ('white', 'black')) Tuple or list containing the two colors of the checkerboard pattern. font_colors : tuple or list (default: ('black', 'white')) Font colors corresponding to the cell colors. figsize : tuple (default: (2.5, 2.5)) Height and width of the figure fmt : str (default: '%.1f') Python string formatter for cell values. The default '%.1f' results in floats with 1 digit after the decimal point. Use '%d' to show numbers as integers. row_labels : list (default: None) List of the row labels. Uses the array row indices 0 to n by default. col_labels : list (default: None) List of the column labels. Uses the array column indices 0 to m by default. fontsize : int (default: None) Specifies the font size of the checkerboard table. Uses matplotlib's default if None. Returns fig : matplotlib Figure object. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/plotting/checkerboard_plot/","title":"checkerboard_plot"},{"location":"api_subpackages/mlxtend.plotting/#ecdf","text":"ecdf(x, y_label='ECDF', x_label=None, ax=None, percentile=None, ecdf_color=None, ecdf_marker='o', percentile_color='black', percentile_linestyle='--') Plots an Empirical Cumulative Distribution Function Parameters x : array or list, shape=[n_samples,] Array-like object containing the feature values y_label : str (default='ECDF') Text label for the y-axis x_label : str (default=None) Text label for the x-axis ax : matplotlib.axes.Axes (default: None) An existing matplotlib Axes. Creates one if ax=None percentile : float (default=None) Float between 0 and 1 for plotting a percentile threshold line ecdf_color : matplotlib color (default=None) Color for the ECDF plot; uses matplotlib defaults if None ecdf_marker : matplotlib marker (default='o') Marker style for the ECDF plot percentile_color : matplotlib color (default='black') Color for the percentile threshold if percentile is not None percentile_linestyle : matplotlib linestyle (default='--') Line style for the percentile threshold if percentile is not None Returns ax : matplotlib.axes.Axes object percentile_threshold : float Feature threshold at the percentile or None if percentile=None percentile_count : Number of if percentile is not None Number of samples that have a feature less or equal than the feature threshold at a percentile threshold or None if percentile=None Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/plotting/ecdf/","title":"ecdf"},{"location":"api_subpackages/mlxtend.plotting/#enrichment_plot","text":"enrichment_plot(df, colors='bgrkcy', markers=' ', linestyles='-', alpha=0.5, lw=2, where='post', grid=True, count_label='Count', xlim='auto', ylim='auto', invert_axes=False, legend_loc='best', ax=None) Plot stacked barplots Parameters df : pandas.DataFrame A pandas DataFrame where columns represent the different categories. colors: str (default: 'bgrcky') The colors of the bars. markers : str (default: ' ') Matplotlib markerstyles, e.g, 'sov' for square,circle, and triangle markers. linestyles : str (default: '-') Matplotlib linestyles, e.g., '-,--' to cycle normal and dashed lines. Note that the different linestyles need to be separated by commas. alpha : float (default: 0.5) Transparency level from 0.0 to 1.0. lw : int or float (default: 2) Linewidth parameter. where : {'post', 'pre', 'mid'} (default: 'post') Starting location of the steps. grid : bool (default: True ) Plots a grid if True. count_label : str (default: 'Count') Label for the \"Count\"-axis. xlim : 'auto' or array-like [min, max] (default: 'auto') Min and maximum position of the x-axis range. ylim : 'auto' or array-like [min, max] (default: 'auto') Min and maximum position of the y-axis range. invert_axes : bool (default: False) Plots count on the x-axis if True. legend_loc : str (default: 'best') Location of the plot legend {best, upper left, upper right, lower left, lower right} No legend if legend_loc=False ax : matplotlib axis, optional (default: None) Use this axis for plotting or make a new one otherwise Returns ax : matplotlib axis Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/plotting/enrichment_plot/","title":"enrichment_plot"},{"location":"api_subpackages/mlxtend.plotting/#heatmap","text":"heatmap(matrix, hide_spines=False, hide_ticks=False, figsize=None, cmap=None, colorbar=True, row_names=None, column_names=None, column_name_rotation=45, cell_values=True, cell_fmt='.2f', cell_font_size=None, text_color_threshold=None) Plot a heatmap via matplotlib. Parameters conf_mat : array-like, shape = [n_rows, n_columns] And arbitrary 2D array. hide_spines : bool (default: False) Hides axis spines if True. hide_ticks : bool (default: False) Hides axis ticks if True figsize : tuple (default: (2.5, 2.5)) Height and width of the figure cmap : matplotlib colormap (default: None ) Uses matplotlib.pyplot.cm.viridis if None colorbar : bool (default: True) Shows a colorbar if True row_names : array-like, shape = [n_rows] (default: None) List of row names to be used as y-axis tick labels. column_names : array-like, shape = [n_columns] (default: None) List of column names to be used as x-axis tick labels. column_name_rotation : int (default: 45) Number of degrees for rotating column x-tick labels. cell_values : bool (default: True) Plots cell values if True. cell_fmt : string (default: '.2f') Format specification for cell values (if cell_values=True ) cell_font_size : int (default: None) Font size for cell values (if cell_values=True ) text_color_threshold : float (default: None) Threshold for the black/white text threshold of the text annotation. Default (None) tried to infer a good threshold automatically using np.max(normed_matrix) / 2 . Returns fig, ax : matplotlib.pyplot subplot objects Figure and axis elements of the subplot. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/plotting/heatmap/","title":"heatmap"},{"location":"api_subpackages/mlxtend.plotting/#plot_confusion_matrix","text":"plot_confusion_matrix(conf_mat, hide_spines=False, hide_ticks=False, figsize=None, cmap=None, colorbar=False, show_absolute=True, show_normed=False, norm_colormap=None, class_names=None, figure=None, axis=None, fontcolor_threshold=0.5) Plot a confusion matrix via matplotlib. Parameters conf_mat : array-like, shape = [n_classes, n_classes] Confusion matrix from evaluate.confusion matrix. hide_spines : bool (default: False) Hides axis spines if True. hide_ticks : bool (default: False) Hides axis ticks if True figsize : tuple (default: (2.5, 2.5)) Height and width of the figure cmap : matplotlib colormap (default: None ) Uses matplotlib.pyplot.cm.Blues if None colorbar : bool (default: False) Shows a colorbar if True show_absolute : bool (default: True) Shows absolute confusion matrix coefficients if True. At least one of show_absolute or show_normed must be True. show_normed : bool (default: False) Shows normed confusion matrix coefficients if True. The normed confusion matrix coefficients give the proportion of training examples per class that are assigned the correct label. At least one of show_absolute or show_normed must be True. norm_colormap : bool (default: False) Matplotlib color normalization object to normalize the color scale, e.g., matplotlib.colors.LogNorm() . class_names : array-like, shape = [n_classes] (default: None) List of class names. If not None , ticks will be set to these values. figure : None or Matplotlib figure (default: None) If None will create a new figure. axis : None or Matplotlib figure axis (default: None) If None will create a new axis. fontcolor_threshold : Float (default: 0.5) Sets a threshold for choosing black and white font colors for the cells. By default all values larger than 0.5 times the maximum cell value are converted to white, and everything equal or smaller than 0.5 times the maximum cell value are converted to black. Returns fig, ax : matplotlib.pyplot subplot objects Figure and axis elements of the subplot. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/plotting/plot_confusion_matrix/","title":"plot_confusion_matrix"},{"location":"api_subpackages/mlxtend.plotting/#plot_decision_regions","text":"plot_decision_regions(X, y, clf, feature_index=None, filler_feature_values=None, filler_feature_ranges=None, ax=None, X_highlight=None, zoom_factor=1.0, legend=1, hide_spines=True, markers='s^oxv<>', colors='#1f77b4,#ff7f0e,#3ca02c,#d62728,#9467bd,#8c564b,#e377c2,#7f7f7f,#bcbd22,#17becf', scatter_kwargs=None, contourf_kwargs=None, contour_kwargs=None, scatter_highlight_kwargs=None, n_jobs=None) Plot decision regions of a classifier. Please note that this functions assumes that class labels are labeled consecutively, e.g,. 0, 1, 2, 3, 4, and 5. If you have class labels with integer labels > 4, you may want to provide additional colors and/or markers as `colors` and `markers` arguments. See https://matplotlib.org/examples/color/named_colors.html for more information. Parameters X : array-like, shape = [n_samples, n_features] Feature Matrix. y : array-like, shape = [n_samples] True class labels. clf : Classifier object. Must have a .predict method. feature_index : array-like (default: (0,) for 1D, (0, 1) otherwise) Feature indices to use for plotting. The first index in feature_index will be on the x-axis, the second index will be on the y-axis. filler_feature_values : dict (default: None) Only needed for number features > 2. Dictionary of feature index-value pairs for the features not being plotted. filler_feature_ranges : dict (default: None) Only needed for number features > 2. Dictionary of feature index-value pairs for the features not being plotted. Will use the ranges provided to select training samples for plotting. ax : matplotlib.axes.Axes (default: None) An existing matplotlib Axes. Creates one if ax=None. X_highlight : array-like, shape = [n_samples, n_features] (default: None) An array with data points that are used to highlight samples in X . zoom_factor : float (default: 1.0) Controls the scale of the x- and y-axis of the decision plot. hide_spines : bool (default: True) Hide axis spines if True. legend : int (default: 1) Integer to specify the legend location. No legend if legend is 0. markers : str (default: 's^oxv<>') Scatterplot markers. colors : str (default: 'red,blue,limegreen,gray,cyan') Comma separated list of colors. scatter_kwargs : dict (default: None) Keyword arguments for underlying matplotlib scatter function. contourf_kwargs : dict (default: None) Keyword arguments for underlying matplotlib contourf function. contour_kwargs : dict (default: None) Keyword arguments for underlying matplotlib contour function (which draws the lines between decision regions). scatter_highlight_kwargs : dict (default: None) Keyword arguments for underlying matplotlib scatter function. n_jobs : int or None, optional (default=None) The number of CPUs to use to do the computation using Python's multiprocessing library. None means 1. -1 means using all processors. New in v0.22.0. Returns ax : matplotlib.axes.Axes object Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/plotting/plot_decision_regions/","title":"plot_decision_regions"},{"location":"api_subpackages/mlxtend.plotting/#plot_learning_curves","text":"plot_learning_curves(X_train, y_train, X_test, y_test, clf, train_marker='o', test_marker='^', scoring='misclassification error', suppress_plot=False, print_model=True, title_fontsize=12, style='default', legend_loc='best') Plots learning curves of a classifier. Parameters X_train : array-like, shape = [n_samples, n_features] Feature matrix of the training dataset. y_train : array-like, shape = [n_samples] True class labels of the training dataset. X_test : array-like, shape = [n_samples, n_features] Feature matrix of the test dataset. y_test : array-like, shape = [n_samples] True class labels of the test dataset. clf : Classifier object. Must have a .predict .fit method. train_marker : str (default: 'o') Marker for the training set line plot. test_marker : str (default: '^') Marker for the test set line plot. scoring : str (default: 'misclassification error') If not 'misclassification error', accepts the following metrics (from scikit-learn): {'accuracy', 'average_precision', 'f1_micro', 'f1_macro', 'f1_weighted', 'f1_samples', 'log_loss', 'precision', 'recall', 'roc_auc', 'adjusted_rand_score', 'mean_absolute_error', 'mean_squared_error', 'median_absolute_error', 'r2'} suppress_plot=False : bool (default: False) Suppress matplotlib plots if True. Recommended for testing purposes. print_model : bool (default: True) Print model parameters in plot title if True. title_fontsize : int (default: 12) Determines the size of the plot title font. style : str (default: 'default') Matplotlib style. For more styles, please see https://matplotlib.org/stable/gallery/style_sheets/style_sheets_reference.html legend_loc : str (default: 'best') Where to place the plot legend: {'best', 'upper left', 'upper right', 'lower left', 'lower right'} Returns errors : (training_error, test_error): tuple of lists Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/plotting/plot_learning_curves/","title":"plot_learning_curves"},{"location":"api_subpackages/mlxtend.plotting/#plot_linear_regression","text":"plot_linear_regression(X, y, model=LinearRegression(), corr_func='pearsonr', scattercolor='blue', fit_style='k--', legend=True, xlim='auto') Plot a linear regression line fit. Parameters X : numpy array, shape = [n_samples,] Samples. y : numpy array, shape (n_samples,) Target values model: object (default: sklearn.linear_model.LinearRegression) Estimator object for regression. Must implement a .fit() and .predict() method. corr_func: str or function (default: 'pearsonr') Uses pearsonr from scipy.stats if corr_func='pearsonr'. to compute the regression slope. If not 'pearsonr', the corr_func , the corr_func parameter expects a function of the form func( , ) as inputs, which is expected to return a tuple (<correlation_coefficient>, <some_unused_value>) . scattercolor: string (default: blue) Color of scatter plot points. fit_style: string (default: k--) Style for the line fit. legend: bool (default: True) Plots legend with corr_coeff coef., fit coef., and intercept values. xlim: array-like (x_min, x_max) or 'auto' (default: 'auto') X-axis limits for the linear line fit. Returns regression_fit : tuple intercept, slope, corr_coeff (float, float, float) Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/plotting/plot_linear_regression/","title":"plot_linear_regression"},{"location":"api_subpackages/mlxtend.plotting/#plot_pca_correlation_graph","text":"plot_pca_correlation_graph(X, variables_names, dimensions=(1, 2), figure_axis_size=6, X_pca=None, explained_variance=None) Compute the PCA for X and plots the Correlation graph Parameters X : 2d array like. The columns represent the different variables and the rows are the samples of thos variables variables_names : array like Name of the columns (the variables) of X dimensions: tuple with two elements. dimensions to be plotted (x,y) figure_axis_size : size of the final frame. The figure created is a square with length and width equal to figure_axis_size. X_pca : np.ndarray, shape = [n_samples, n_components]. Optional. X_pca is the matrix of the transformed components from X. If not provided, the function computes PCA automatically using mlxtend.feature_extraction.PrincipalComponentAnalysis Expected n_componentes >= max(dimensions) explained_variance : 1 dimension np.ndarray, length = n_components Optional. explained_variance are the eigenvalues from the diagonalized covariance matrix on the PCA transformatiopn. If not provided, the function computes PCA independently Expected n_componentes == X.shape[1] Returns matplotlib_figure, correlation_matrix Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/plotting/plot_pca_correlation_graph/","title":"plot_pca_correlation_graph"},{"location":"api_subpackages/mlxtend.plotting/#plot_sequential_feature_selection","text":"plot_sequential_feature_selection(metric_dict, figsize=None, kind='std_dev', color='blue', bcolor='steelblue', marker='o', alpha=0.2, ylabel='Performance', confidence_interval=0.95) Plot feature selection results. Parameters metric_dict : mlxtend.SequentialFeatureSelector.get_metric_dict() object figsize : tuple (default: None) Height and width of the figure kind : str (default: \"std_dev\") The kind of error bar or confidence interval in {'std_dev', 'std_err', 'ci', None}. color : str (default: \"blue\") Color of the lineplot (accepts any matplotlib color name) bcolor : str (default: \"steelblue\"). Color of the error bars / confidence intervals (accepts any matplotlib color name). marker : str (default: \"o\") Marker of the line plot (accepts any matplotlib marker name). alpha : float in [0, 1] (default: 0.2) Transparency of the error bars / confidence intervals. ylabel : str (default: \"Performance\") Y-axis label. confidence_interval : float (default: 0.95) Confidence level if kind='ci' . Returns fig : matplotlib.pyplot.figure() object Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/plotting/plot_sequential_feature_selection/","title":"plot_sequential_feature_selection"},{"location":"api_subpackages/mlxtend.plotting/#remove_borders","text":"remove_borders(axes, left=False, bottom=False, right=True, top=True) Remove chart junk from matplotlib plots. Parameters axes : iterable An iterable containing plt.gca() or plt.subplot() objects, e.g. [plt.gca()]. left : bool (default: False ) Hide left axis spine if True. bottom : bool (default: False ) Hide bottom axis spine if True. right : bool (default: True ) Hide right axis spine if True. top : bool (default: True ) Hide top axis spine if True. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/plotting/remove_chartjunk/","title":"remove_borders"},{"location":"api_subpackages/mlxtend.plotting/#scatter_hist","text":"scatter_hist(x, y, xlabel=None, ylabel=None, figsize=(5, 5)) Scatter plot and individual feature histograms along axes. Parameters x : 1D array-like or Pandas Series X-axis values. y : 1D array-like or Pandas Series Y-axis values. xlabel : str (default: None ) Label for the X-axis values. If x is a pandas Series, and xlabel is None , the label is inferred automatically. ylabel : str (default: None ) Label for the X-axis values. If y is a pandas Series, and ylabel is None , the label is inferred automatically. figsize : tuple (default: (5, 5) ) Matplotlib figure size. Returns plot : Matplotlib Figure object","title":"scatter_hist"},{"location":"api_subpackages/mlxtend.plotting/#scatterplotmatrix","text":"scatterplotmatrix(X, fig_axes=None, names=None, figsize=(8, 8), alpha=1.0, kwargs) Lower triangular of a scatterplot matrix Parameters X : array-like, shape={num_examples, num_features} Design matrix containing data instances (examples) with multiple exploratory variables (features). fix_axes : tuple (default: None) A (fig, axes) tuple, where fig is an figure object and axes is an axes object created via matplotlib, for example, by calling the pyplot subplot function fig, axes = plt.subplots(...) names : list (default: None) A list of string names, which should have the same number of elements as there are features (columns) in X . figsize : tuple (default: (8, 8)) Height and width of the subplot grid. Ignored if fig_axes is not None . alpha : float (default: 1.0) Transparency for both the scatter plots and the histograms along the diagonal. **kwargs : kwargs Keyword arguments for the scatterplots. Returns fix_axes : tuple A (fig, axes) tuple, where fig is an figure object and axes is an axes object created via matplotlib, for example, by calling the pyplot subplot function fig, axes = plt.subplots(...) Examples For more usage examples, please see https://rasbt.github.io/mlxtend/user_guide/plotting/scatterplotmatrix/","title":"scatterplotmatrix"},{"location":"api_subpackages/mlxtend.plotting/#stacked_barplot","text":"stacked_barplot(df, bar_width='auto', colors='bgrcky', labels='index', rotation=90, legend_loc='best') Function to plot stacked barplots Parameters df : pandas.DataFrame A pandas DataFrame where the index denotes the x-axis labels, and the columns contain the different measurements for each row. bar_width: 'auto' or float (default: 'auto') Parameter to set the widths of the bars. if 'auto', the width is automatically determined by the number of columns in the dataset. colors: str (default: 'bgrcky') The colors of the bars. labels: 'index' or iterable (default: 'index') If 'index', the DataFrame index will be used as x-tick labels. rotation: int (default: 90) Parameter to rotate the x-axis labels. legend_loc : str (default: 'best') Location of the plot legend {best, upper left, upper right, lower left, lower right} No legend if legend_loc=False Returns fig : matplotlib.pyplot figure object Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/plotting/stacked_barplot/","title":"stacked_barplot"},{"location":"api_subpackages/mlxtend.preprocessing/","text":"mlxtend version: 0.23.1 CopyTransformer CopyTransformer() Transformer that returns a copy of the input array For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/preprocessing/CopyTransformer/ Methods fit(X, y=None) Mock method. Does nothing. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] (default: None) Returns self fit_transform(X, y=None) Return a copy of the input array. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] (default: None) Returns X_copy : copy of the input X array. get_params(deep=True) Get parameters for this estimator. Parameters deep : bool, default=True If True, will return the parameters for this estimator and contained subobjects that are estimators. Returns params : dict Parameter names mapped to their values. set_params( params) Set the parameters of this estimator. The method works on simple estimators as well as on nested objects (such as :class:`~sklearn.pipeline.Pipeline`). The latter have parameters of the form ``<component>__<parameter>`` so that it's possible to update each component of a nested object. Parameters **params : dict Estimator parameters. Returns self : estimator instance Estimator instance. transform(X, y=None) Return a copy of the input array. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] (default: None) Returns X_copy : copy of the input X array. DenseTransformer DenseTransformer(return_copy=True) Convert a sparse array into a dense array. For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/preprocessing/DenseTransformer/ Methods fit(X, y=None) Mock method. Does nothing. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] (default: None) Returns self fit_transform(X, y=None) Return a dense version of the input array. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] (default: None) Returns X_dense : dense version of the input X array. get_params(deep=True) Get parameters for this estimator. Parameters deep : bool, default=True If True, will return the parameters for this estimator and contained subobjects that are estimators. Returns params : dict Parameter names mapped to their values. set_params( params) Set the parameters of this estimator. The method works on simple estimators as well as on nested objects (such as :class:`~sklearn.pipeline.Pipeline`). The latter have parameters of the form ``<component>__<parameter>`` so that it's possible to update each component of a nested object. Parameters **params : dict Estimator parameters. Returns self : estimator instance Estimator instance. transform(X, y=None) Return a dense version of the input array. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] (default: None) Returns X_dense : dense version of the input X array. MeanCenterer MeanCenterer() Column centering of vectors and matrices. Attributes col_means : numpy.ndarray [n_columns] NumPy array storing the mean values for centering after fitting the MeanCenterer object. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/preprocessing/MeanCenterer/ Methods fit(X) Gets the column means for mean centering. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Array of data vectors, where n_samples is the number of samples and n_features is the number of features. Returns self fit_transform(X) Fits and transforms an arry. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Array of data vectors, where n_samples is the number of samples and n_features is the number of features. Returns X_tr : {array-like, sparse matrix}, shape = [n_samples, n_features] A copy of the input array with the columns centered. transform(X) Centers a NumPy array. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Array of data vectors, where n_samples is the number of samples and n_features is the number of features. Returns X_tr : {array-like, sparse matrix}, shape = [n_samples, n_features] A copy of the input array with the columns centered. TransactionEncoder TransactionEncoder() Encoder class for transaction data in Python lists Parameters None Attributes columns_: list List of unique names in the X input list of lists Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/preprocessing/TransactionEncoder/ Methods fit(X) Learn unique column names from transaction DataFrame Parameters X : list of lists A python list of lists, where the outer list stores the n transactions and the inner list stores the items in each transaction. For example, [['Apple', 'Beer', 'Rice', 'Chicken'], ['Apple', 'Beer', 'Rice'], ['Apple', 'Beer'], ['Apple', 'Bananas'], ['Milk', 'Beer', 'Rice', 'Chicken'], ['Milk', 'Beer', 'Rice'], ['Milk', 'Beer'], ['Apple', 'Bananas']] fit_transform(X, sparse=False) Fit a TransactionEncoder encoder and transform a dataset. get_params(deep=True) Get parameters for this estimator. Parameters deep : bool, default=True If True, will return the parameters for this estimator and contained subobjects that are estimators. Returns params : dict Parameter names mapped to their values. inverse_transform(array) Transforms an encoded NumPy array back into transactions. Parameters array : NumPy array [n_transactions, n_unique_items] The NumPy one-hot encoded boolean array of the input transactions, where the columns represent the unique items found in the input array in alphabetic order For example, array([[True , False, True , True , False, True ], [True , False, True , False, False, True ], [True , False, True , False, False, False], [True , True , False, False, False, False], [False, False, True , True , True , True ], [False, False, True , False, True , True ], [False, False, True , False, True , False], [True , True , False, False, False, False]]) The corresponding column labels are available as self.columns_, e.g., ['Apple', 'Bananas', 'Beer', 'Chicken', 'Milk', 'Rice'] Returns X : list of lists A python list of lists, where the outer list stores the n transactions and the inner list stores the items in each transaction. For example, [['Apple', 'Beer', 'Rice', 'Chicken'], ['Apple', 'Beer', 'Rice'], ['Apple', 'Beer'], ['Apple', 'Bananas'], ['Milk', 'Beer', 'Rice', 'Chicken'], ['Milk', 'Beer', 'Rice'], ['Milk', 'Beer'], ['Apple', 'Bananas']] set_output( , transform=None)* Set output container. See :ref:`sphx_glr_auto_examples_miscellaneous_plot_set_output.py` for an example on how to use the API. Parameters transform : {\"default\", \"pandas\"}, default=None Configure output of transform and fit_transform . \"default\" : Default output format of a transformer \"pandas\" : DataFrame output None : Transform configuration is unchanged Returns self : estimator instance Estimator instance. set_params( params) Set the parameters of this estimator. The method works on simple estimators as well as on nested objects (such as :class:`~sklearn.pipeline.Pipeline`). The latter have parameters of the form ``<component>__<parameter>`` so that it's possible to update each component of a nested object. Parameters **params : dict Estimator parameters. Returns self : estimator instance Estimator instance. transform(X, sparse=False) Transform transactions into a one-hot encoded NumPy array. Parameters X : list of lists A python list of lists, where the outer list stores the n transactions and the inner list stores the items in each transaction. For example, [['Apple', 'Beer', 'Rice', 'Chicken'], ['Apple', 'Beer', 'Rice'], ['Apple', 'Beer'], ['Apple', 'Bananas'], ['Milk', 'Beer', 'Rice', 'Chicken'], ['Milk', 'Beer', 'Rice'], ['Milk', 'Beer'], ['Apple', 'Bananas']] sparse: bool (default=False) If True, transform will return Compressed Sparse Row matrix instead of the regular one. Returns array : NumPy array [n_transactions, n_unique_items] if sparse=False (default). Compressed Sparse Row matrix otherwise The one-hot encoded boolean array of the input transactions, where the columns represent the unique items found in the input array in alphabetic order. Exact representation depends on the sparse argument For example, array([[True , False, True , True , False, True ], [True , False, True , False, False, True ], [True , False, True , False, False, False], [True , True , False, False, False, False], [False, False, True , True , True , True ], [False, False, True , False, True , True ], [False, False, True , False, True , False], [True , True , False, False, False, False]]) The corresponding column labels are available as self.columns_, e.g., ['Apple', 'Bananas', 'Beer', 'Chicken', 'Milk', 'Rice'] minmax_scaling minmax_scaling(array, columns, min_val=0, max_val=1) Min max scaling of pandas' DataFrames. Parameters array : pandas DataFrame or NumPy ndarray, shape = [n_rows, n_columns]. columns : array-like, shape = [n_columns] Array-like with column names, e.g., ['col1', 'col2', ...] or column indices [0, 2, 4, ...] min_val : int or float , optional (default= 0 ) minimum value after rescaling. max_val : int or float , optional (default= 1 ) maximum value after rescaling. Returns df_new : pandas DataFrame object. Copy of the array or DataFrame with rescaled columns. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/preprocessing/minmax_scaling/ one_hot one_hot(y, num_labels='auto', dtype='float') One-hot encoding of class labels Parameters y : array-like, shape = [n_classlabels] Python list or numpy array consisting of class labels. num_labels : int or 'auto' Number of unique labels in the class label array. Infers the number of unique labels from the input array if set to 'auto'. dtype : str NumPy array type (float, float32, float64) of the output array. Returns ary : numpy.ndarray, shape = [n_classlabels] One-hot encoded array, where each sample is represented as a row vector in the returned array. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/preprocessing/one_hot/ shuffle_arrays_unison shuffle_arrays_unison(arrays, random_seed=None) Shuffle NumPy arrays in unison. Parameters arrays : array-like, shape = [n_arrays] A list of NumPy arrays. random_seed : int (default: None) Sets the random state. Returns shuffled_arrays : A list of NumPy arrays after shuffling. Examples ``` >>> import numpy as np >>> from mlxtend.preprocessing import shuffle_arrays_unison >>> X1 = np.array([[1, 2, 3], [4, 5, 6], [7, 8, 9]]) >>> y1 = np.array([1, 2, 3]) >>> X2, y2 = shuffle_arrays_unison(arrays=[X1, y1], random_seed=3) >>> assert(X2.all() == np.array([[4, 5, 6], [1, 2, 3], [7, 8, 9]]).all()) >>> assert(y2.all() == np.array([2, 1, 3]).all()) >>> For more usage examples, please see https://rasbt.github.io/mlxtend/user_guide/preprocessing/shuffle_arrays_unison/ ``` standardize standardize(array, columns=None, ddof=0, return_params=False, params=None) Standardize columns in pandas DataFrames. Parameters array : pandas DataFrame or NumPy ndarray, shape = [n_rows, n_columns]. columns : array-like, shape = [n_columns] (default: None) Array-like with column names, e.g., ['col1', 'col2', ...] or column indices [0, 2, 4, ...] If None, standardizes all columns. ddof : int (default: 0) Delta Degrees of Freedom. The divisor used in calculations is N - ddof, where N represents the number of elements. return_params : dict (default: False) If set to True, a dictionary is returned in addition to the standardized array. The parameter dictionary contains the column means ('avgs') and standard deviations ('stds') of the individual columns. params : dict (default: None) A dictionary with column means and standard deviations as returned by the standardize function if return_params was set to True. If a params dictionary is provided, the standardize function will use these instead of computing them from the current array. Notes If all values in a given column are the same, these values are all set to 0.0 . The standard deviation in the parameters dictionary is consequently set to 1.0 to avoid dividing by zero. Returns df_new : pandas DataFrame object. Copy of the array or DataFrame with standardized columns. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/preprocessing/standardize/","title":"Mlxtend.preprocessing"},{"location":"api_subpackages/mlxtend.preprocessing/#copytransformer","text":"CopyTransformer() Transformer that returns a copy of the input array For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/preprocessing/CopyTransformer/","title":"CopyTransformer"},{"location":"api_subpackages/mlxtend.preprocessing/#methods","text":"fit(X, y=None) Mock method. Does nothing. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] (default: None) Returns self fit_transform(X, y=None) Return a copy of the input array. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] (default: None) Returns X_copy : copy of the input X array. get_params(deep=True) Get parameters for this estimator. Parameters deep : bool, default=True If True, will return the parameters for this estimator and contained subobjects that are estimators. Returns params : dict Parameter names mapped to their values. set_params( params) Set the parameters of this estimator. The method works on simple estimators as well as on nested objects (such as :class:`~sklearn.pipeline.Pipeline`). The latter have parameters of the form ``<component>__<parameter>`` so that it's possible to update each component of a nested object. Parameters **params : dict Estimator parameters. Returns self : estimator instance Estimator instance. transform(X, y=None) Return a copy of the input array. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] (default: None) Returns X_copy : copy of the input X array.","title":"Methods"},{"location":"api_subpackages/mlxtend.preprocessing/#densetransformer","text":"DenseTransformer(return_copy=True) Convert a sparse array into a dense array. For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/preprocessing/DenseTransformer/","title":"DenseTransformer"},{"location":"api_subpackages/mlxtend.preprocessing/#methods_1","text":"fit(X, y=None) Mock method. Does nothing. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] (default: None) Returns self fit_transform(X, y=None) Return a dense version of the input array. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] (default: None) Returns X_dense : dense version of the input X array. get_params(deep=True) Get parameters for this estimator. Parameters deep : bool, default=True If True, will return the parameters for this estimator and contained subobjects that are estimators. Returns params : dict Parameter names mapped to their values. set_params( params) Set the parameters of this estimator. The method works on simple estimators as well as on nested objects (such as :class:`~sklearn.pipeline.Pipeline`). The latter have parameters of the form ``<component>__<parameter>`` so that it's possible to update each component of a nested object. Parameters **params : dict Estimator parameters. Returns self : estimator instance Estimator instance. transform(X, y=None) Return a dense version of the input array. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] (default: None) Returns X_dense : dense version of the input X array.","title":"Methods"},{"location":"api_subpackages/mlxtend.preprocessing/#meancenterer","text":"MeanCenterer() Column centering of vectors and matrices. Attributes col_means : numpy.ndarray [n_columns] NumPy array storing the mean values for centering after fitting the MeanCenterer object. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/preprocessing/MeanCenterer/","title":"MeanCenterer"},{"location":"api_subpackages/mlxtend.preprocessing/#methods_2","text":"fit(X) Gets the column means for mean centering. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Array of data vectors, where n_samples is the number of samples and n_features is the number of features. Returns self fit_transform(X) Fits and transforms an arry. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Array of data vectors, where n_samples is the number of samples and n_features is the number of features. Returns X_tr : {array-like, sparse matrix}, shape = [n_samples, n_features] A copy of the input array with the columns centered. transform(X) Centers a NumPy array. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Array of data vectors, where n_samples is the number of samples and n_features is the number of features. Returns X_tr : {array-like, sparse matrix}, shape = [n_samples, n_features] A copy of the input array with the columns centered.","title":"Methods"},{"location":"api_subpackages/mlxtend.preprocessing/#transactionencoder","text":"TransactionEncoder() Encoder class for transaction data in Python lists Parameters None Attributes columns_: list List of unique names in the X input list of lists Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/preprocessing/TransactionEncoder/","title":"TransactionEncoder"},{"location":"api_subpackages/mlxtend.preprocessing/#methods_3","text":"fit(X) Learn unique column names from transaction DataFrame Parameters X : list of lists A python list of lists, where the outer list stores the n transactions and the inner list stores the items in each transaction. For example, [['Apple', 'Beer', 'Rice', 'Chicken'], ['Apple', 'Beer', 'Rice'], ['Apple', 'Beer'], ['Apple', 'Bananas'], ['Milk', 'Beer', 'Rice', 'Chicken'], ['Milk', 'Beer', 'Rice'], ['Milk', 'Beer'], ['Apple', 'Bananas']] fit_transform(X, sparse=False) Fit a TransactionEncoder encoder and transform a dataset. get_params(deep=True) Get parameters for this estimator. Parameters deep : bool, default=True If True, will return the parameters for this estimator and contained subobjects that are estimators. Returns params : dict Parameter names mapped to their values. inverse_transform(array) Transforms an encoded NumPy array back into transactions. Parameters array : NumPy array [n_transactions, n_unique_items] The NumPy one-hot encoded boolean array of the input transactions, where the columns represent the unique items found in the input array in alphabetic order For example, array([[True , False, True , True , False, True ], [True , False, True , False, False, True ], [True , False, True , False, False, False], [True , True , False, False, False, False], [False, False, True , True , True , True ], [False, False, True , False, True , True ], [False, False, True , False, True , False], [True , True , False, False, False, False]]) The corresponding column labels are available as self.columns_, e.g., ['Apple', 'Bananas', 'Beer', 'Chicken', 'Milk', 'Rice'] Returns X : list of lists A python list of lists, where the outer list stores the n transactions and the inner list stores the items in each transaction. For example, [['Apple', 'Beer', 'Rice', 'Chicken'], ['Apple', 'Beer', 'Rice'], ['Apple', 'Beer'], ['Apple', 'Bananas'], ['Milk', 'Beer', 'Rice', 'Chicken'], ['Milk', 'Beer', 'Rice'], ['Milk', 'Beer'], ['Apple', 'Bananas']] set_output( , transform=None)* Set output container. See :ref:`sphx_glr_auto_examples_miscellaneous_plot_set_output.py` for an example on how to use the API. Parameters transform : {\"default\", \"pandas\"}, default=None Configure output of transform and fit_transform . \"default\" : Default output format of a transformer \"pandas\" : DataFrame output None : Transform configuration is unchanged Returns self : estimator instance Estimator instance. set_params( params) Set the parameters of this estimator. The method works on simple estimators as well as on nested objects (such as :class:`~sklearn.pipeline.Pipeline`). The latter have parameters of the form ``<component>__<parameter>`` so that it's possible to update each component of a nested object. Parameters **params : dict Estimator parameters. Returns self : estimator instance Estimator instance. transform(X, sparse=False) Transform transactions into a one-hot encoded NumPy array. Parameters X : list of lists A python list of lists, where the outer list stores the n transactions and the inner list stores the items in each transaction. For example, [['Apple', 'Beer', 'Rice', 'Chicken'], ['Apple', 'Beer', 'Rice'], ['Apple', 'Beer'], ['Apple', 'Bananas'], ['Milk', 'Beer', 'Rice', 'Chicken'], ['Milk', 'Beer', 'Rice'], ['Milk', 'Beer'], ['Apple', 'Bananas']] sparse: bool (default=False) If True, transform will return Compressed Sparse Row matrix instead of the regular one. Returns array : NumPy array [n_transactions, n_unique_items] if sparse=False (default). Compressed Sparse Row matrix otherwise The one-hot encoded boolean array of the input transactions, where the columns represent the unique items found in the input array in alphabetic order. Exact representation depends on the sparse argument For example, array([[True , False, True , True , False, True ], [True , False, True , False, False, True ], [True , False, True , False, False, False], [True , True , False, False, False, False], [False, False, True , True , True , True ], [False, False, True , False, True , True ], [False, False, True , False, True , False], [True , True , False, False, False, False]]) The corresponding column labels are available as self.columns_, e.g., ['Apple', 'Bananas', 'Beer', 'Chicken', 'Milk', 'Rice']","title":"Methods"},{"location":"api_subpackages/mlxtend.preprocessing/#minmax_scaling","text":"minmax_scaling(array, columns, min_val=0, max_val=1) Min max scaling of pandas' DataFrames. Parameters array : pandas DataFrame or NumPy ndarray, shape = [n_rows, n_columns]. columns : array-like, shape = [n_columns] Array-like with column names, e.g., ['col1', 'col2', ...] or column indices [0, 2, 4, ...] min_val : int or float , optional (default= 0 ) minimum value after rescaling. max_val : int or float , optional (default= 1 ) maximum value after rescaling. Returns df_new : pandas DataFrame object. Copy of the array or DataFrame with rescaled columns. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/preprocessing/minmax_scaling/","title":"minmax_scaling"},{"location":"api_subpackages/mlxtend.preprocessing/#one_hot","text":"one_hot(y, num_labels='auto', dtype='float') One-hot encoding of class labels Parameters y : array-like, shape = [n_classlabels] Python list or numpy array consisting of class labels. num_labels : int or 'auto' Number of unique labels in the class label array. Infers the number of unique labels from the input array if set to 'auto'. dtype : str NumPy array type (float, float32, float64) of the output array. Returns ary : numpy.ndarray, shape = [n_classlabels] One-hot encoded array, where each sample is represented as a row vector in the returned array. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/preprocessing/one_hot/","title":"one_hot"},{"location":"api_subpackages/mlxtend.preprocessing/#shuffle_arrays_unison","text":"shuffle_arrays_unison(arrays, random_seed=None) Shuffle NumPy arrays in unison. Parameters arrays : array-like, shape = [n_arrays] A list of NumPy arrays. random_seed : int (default: None) Sets the random state. Returns shuffled_arrays : A list of NumPy arrays after shuffling. Examples ``` >>> import numpy as np >>> from mlxtend.preprocessing import shuffle_arrays_unison >>> X1 = np.array([[1, 2, 3], [4, 5, 6], [7, 8, 9]]) >>> y1 = np.array([1, 2, 3]) >>> X2, y2 = shuffle_arrays_unison(arrays=[X1, y1], random_seed=3) >>> assert(X2.all() == np.array([[4, 5, 6], [1, 2, 3], [7, 8, 9]]).all()) >>> assert(y2.all() == np.array([2, 1, 3]).all()) >>> For more usage examples, please see https://rasbt.github.io/mlxtend/user_guide/preprocessing/shuffle_arrays_unison/ ```","title":"shuffle_arrays_unison"},{"location":"api_subpackages/mlxtend.preprocessing/#standardize","text":"standardize(array, columns=None, ddof=0, return_params=False, params=None) Standardize columns in pandas DataFrames. Parameters array : pandas DataFrame or NumPy ndarray, shape = [n_rows, n_columns]. columns : array-like, shape = [n_columns] (default: None) Array-like with column names, e.g., ['col1', 'col2', ...] or column indices [0, 2, 4, ...] If None, standardizes all columns. ddof : int (default: 0) Delta Degrees of Freedom. The divisor used in calculations is N - ddof, where N represents the number of elements. return_params : dict (default: False) If set to True, a dictionary is returned in addition to the standardized array. The parameter dictionary contains the column means ('avgs') and standard deviations ('stds') of the individual columns. params : dict (default: None) A dictionary with column means and standard deviations as returned by the standardize function if return_params was set to True. If a params dictionary is provided, the standardize function will use these instead of computing them from the current array. Notes If all values in a given column are the same, these values are all set to 0.0 . The standard deviation in the parameters dictionary is consequently set to 1.0 to avoid dividing by zero. Returns df_new : pandas DataFrame object. Copy of the array or DataFrame with standardized columns. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/preprocessing/standardize/","title":"standardize"},{"location":"api_subpackages/mlxtend.regressor/","text":"mlxtend version: 0.23.1 LinearRegression LinearRegression(method='direct', eta=0.01, epochs=50, minibatches=None, random_seed=None, print_progress=0) Ordinary least squares linear regression. Parameters method : string (default: 'direct') For gradient descent-based optimization, use sgd (see minibatch parameter for further options). Otherwise, if direct (default), the analytical method is used. For alternative, numerically more stable solutions, use either qr (QR decomopisition) or svd (Singular Value Decomposition). eta : float (default: 0.01) solver learning rate (between 0.0 and 1.0). Used with method = 'sgd' . (See methods parameter for details) epochs : int (default: 50) Passes over the training dataset. Prior to each epoch, the dataset is shuffled if minibatches > 1 to prevent cycles in stochastic gradient descent. Used with method = 'sgd' . (See methods parameter for details) minibatches : int (default: None) The number of minibatches for gradient-based optimization. If None: Direct method, QR, or SVD method (see method parameter for details) If 1: Gradient Descent learning If len(y): Stochastic Gradient Descent learning If 1 < minibatches < len(y): Minibatch learning random_seed : int (default: None) Set random state for shuffling and initializing the weights. Used in method = 'sgd' . (See methods parameter for details) print_progress : int (default: 0) Prints progress in fitting to stderr if method = 'sgd' . 0: No output 1: Epochs elapsed and cost 2: 1 plus time elapsed 3: 2 plus estimated time until completion Attributes w_ : 2d-array, shape={n_features, 1} Model weights after fitting. b_ : 1d-array, shape={1,} Bias unit after fitting. cost_ : list Sum of squared errors after each epoch; ignored if solver='normal equation' Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/regressor/LinearRegression/ Methods fit(X, y, init_params=True) Learn model from training data. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values. init_params : bool (default: True) Re-initializes model parameters prior to fitting. Set False to continue training with weights from a previous model fitting. Returns self : object get_params(deep=True) Get parameters for this estimator. Parameters deep : boolean, optional If True, will return the parameters for this estimator and contained subobjects that are estimators. Returns params : mapping of string to any Parameter names mapped to their values.' adapted from https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/base.py Author: Gael Varoquaux gael.varoquaux@normalesup.org License: BSD 3 clause predict(X) Predict targets from X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns target_values : array-like, shape = [n_samples] Predicted target values. set_params( params) Set the parameters of this estimator. The method works on simple estimators as well as on nested objects (such as pipelines). The latter have parameters of the form <component>__<parameter> so that it's possible to update each component of a nested object. Returns self adapted from https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/base.py Author: Gael Varoquaux <gael.varoquaux@normalesup.org> License: BSD 3 clause StackingCVRegressor StackingCVRegressor(regressors, meta_regressor, cv=5, shuffle=True, random_state=None, verbose=0, refit=True, use_features_in_secondary=False, store_train_meta_features=False, n_jobs=None, pre_dispatch='2 n_jobs', multi_output=False)* A 'Stacking Cross-Validation' regressor for scikit-learn estimators. Parameters regressors : array-like, shape = [n_regressors] A list of regressors. Invoking the fit method on the StackingCVRegressor will fit clones of these original regressors that will be stored in the class attribute self.regr_ . meta_regressor : object The meta-regressor to be fitted on the ensemble of regressor cv : int, cross-validation generator or iterable, optional (default: 5) Determines the cross-validation splitting strategy. Possible inputs for cv are: - None, to use the default 5-fold cross validation, - integer, to specify the number of folds in a KFold , - An object to be used as a cross-validation generator. - An iterable yielding train, test splits. For integer/None inputs, it will use KFold cross-validation shuffle : bool (default: True) If True, and the cv argument is integer, the training data will be shuffled at fitting stage prior to cross-validation. If the cv argument is a specific cross validation technique, this argument is omitted. random_state : int, RandomState instance or None, optional (default: None) Constrols the randomness of the cv splitter. Used when cv is integer and shuffle=True . New in v0.16.0. verbose : int, optional (default=0) Controls the verbosity of the building process. New in v0.16.0 refit : bool (default: True) Clones the regressors for stacking regression if True (default) or else uses the original ones, which will be refitted on the dataset upon calling the fit method. Setting refit=False is recommended if you are working with estimators that are supporting the scikit-learn fit/predict API interface but are not compatible to scikit-learn's clone function. use_features_in_secondary : bool (default: False) If True, the meta-regressor will be trained both on the predictions of the original regressors and the original dataset. If False, the meta-regressor will be trained only on the predictions of the original regressors. store_train_meta_features : bool (default: False) If True, the meta-features computed from the training data used for fitting the meta-regressor stored in the self.train_meta_features_ array, which can be accessed after calling fit . n_jobs : int or None, optional (default=None) The number of CPUs to use to do the computation. None means 1 unless in a :obj: joblib.parallel_backend context. -1 means using all processors. See :term: Glossary <n_jobs> for more details. New in v0.16.0. pre_dispatch : int, or string, optional Controls the number of jobs that get dispatched during parallel execution. Reducing this number can be useful to avoid an explosion of memory consumption when more jobs get dispatched than CPUs can process. This parameter can be: - None, in which case all the jobs are immediately created and spawned. Use this for lightweight and fast-running jobs, to avoid delays due to on-demand spawning of the jobs - An int, giving the exact number of total jobs that are spawned - A string, giving an expression as a function of n_jobs, as in '2*n_jobs' multi_output : bool (default: False) If True, allow multi-output targets, but forbid nan or inf values. If False, y will be checked to be a vector. (New in v0.19.0.) Attributes train_meta_features : numpy array, shape = [n_samples, n_regressors] meta-features for training data, where n_samples is the number of samples in training data and len(self.regressors) is the number of regressors. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/regressor/StackingCVRegressor/ Methods fit(X, y, groups=None, sample_weight=None) Fit ensemble regressors and the meta-regressor. Parameters X : numpy array, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : numpy array, shape = [n_samples] or [n_samples, n_targets] Target values. Multiple targets are supported only if self.multi_output is True. groups : numpy array/None, shape = [n_samples] The group that each sample belongs to. This is used by specific folding strategies such as GroupKFold() sample_weight : array-like, shape = [n_samples], optional Sample weights passed as sample_weights to each regressor in the regressors list as well as the meta_regressor. Raises error if some regressor does not support sample_weight in the fit() method. Returns self : object fit_transform(X, y=None, fit_params) Fit to data, then transform it. Fits transformer to `X` and `y` with optional parameters `fit_params` and returns a transformed version of `X`. Parameters X : array-like of shape (n_samples, n_features) Input samples. y : array-like of shape (n_samples,) or (n_samples, n_outputs), default=None Target values (None for unsupervised transformations). **fit_params : dict Additional fit parameters. Returns X_new : ndarray array of shape (n_samples, n_features_new) Transformed array. get_params(deep=True) Get parameters for this estimator. Parameters deep : bool, default=True If True, will return the parameters for this estimator and contained subobjects that are estimators. Returns params : dict Parameter names mapped to their values. predict(X) Predict target values for X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns y_target : array-like, shape = [n_samples] or [n_samples, n_targets] Predicted target values. predict_meta_features(X) Get meta-features of test-data. Parameters X : numpy array, shape = [n_samples, n_features] Test vectors, where n_samples is the number of samples and n_features is the number of features. Returns meta-features : numpy array, shape = [n_samples, len(self.regressors)] meta-features for test data, where n_samples is the number of samples in test data and len(self.regressors) is the number of regressors. If self.multi_output is True, then the number of columns is len(self.regressors) * n_targets. score(X, y, sample_weight=None) Return the coefficient of determination of the prediction. The coefficient of determination :math:`R^2` is defined as :math:`(1 - \\frac{u}{v})`, where :math:`u` is the residual sum of squares ((y_true - y_pred)** 2).sum() and :math: v is the total sum of squares ((y_true - y_true.mean()) ** 2).sum() . The best possible score is 1.0 and it can be negative (because the model can be arbitrarily worse). A constant model that always predicts the expected value of y , disregarding the input features, would get a :math: R^2 score of 0.0. Parameters X : array-like of shape (n_samples, n_features) Test samples. For some estimators this may be a precomputed kernel matrix or a list of generic objects instead with shape (n_samples, n_samples_fitted) , where n_samples_fitted is the number of samples used in the fitting for the estimator. y : array-like of shape (n_samples,) or (n_samples, n_outputs) True values for X . sample_weight : array-like of shape (n_samples,), default=None Sample weights. Returns score : float :math: R^2 of self.predict(X) w.r.t. y . Notes The :math: R^2 score used when calling score on a regressor uses multioutput='uniform_average' from version 0.23 to keep consistent with default value of :func: ~sklearn.metrics.r2_score . This influences the score method of all the multioutput regressors (except for :class: ~sklearn.multioutput.MultiOutputRegressor ). set_output( , transform=None)* Set output container. See :ref:`sphx_glr_auto_examples_miscellaneous_plot_set_output.py` for an example on how to use the API. Parameters transform : {\"default\", \"pandas\"}, default=None Configure output of transform and fit_transform . \"default\" : Default output format of a transformer \"pandas\" : DataFrame output None : Transform configuration is unchanged Returns self : estimator instance Estimator instance. set_params( params) Set the parameters of this estimator. Valid parameter keys can be listed with ``get_params()``. Returns self Properties named_regressors Returns List of named estimator tuples, like [('svc', SVC(...))] StackingRegressor StackingRegressor(regressors, meta_regressor, verbose=0, use_features_in_secondary=False, store_train_meta_features=False, refit=True, multi_output=False) A Stacking regressor for scikit-learn estimators for regression. Parameters regressors : array-like, shape = [n_regressors] A list of regressors. Invoking the fit method on the StackingRegressor will fit clones of those original regressors that will be stored in the class attribute self.regr_ . meta_regressor : object The meta-regressor to be fitted on the ensemble of regressors verbose : int, optional (default=0) Controls the verbosity of the building process. - verbose=0 (default): Prints nothing - verbose=1 : Prints the number & name of the regressor being fitted - verbose=2 : Prints info about the parameters of the regressor being fitted - verbose>2 : Changes verbose param of the underlying regressor to self.verbose - 2 use_features_in_secondary : bool (default: False) If True, the meta-regressor will be trained both on the predictions of the original regressors and the original dataset. If False, the meta-regressor will be trained only on the predictions of the original regressors. store_train_meta_features : bool (default: False) If True, the meta-features computed from the training data used for fitting the meta-regressor stored in the self.train_meta_features_ array, which can be accessed after calling fit . Attributes regr_ : list, shape=[n_regressors] Fitted regressors (clones of the original regressors) meta_regr_ : estimator Fitted meta-regressor (clone of the original meta-estimator) coef_ : array-like, shape = [n_features] Model coefficients of the fitted meta-estimator intercept_ : float Intercept of the fitted meta-estimator train_meta_features : numpy array, shape = [n_samples, len(self.regressors)] meta-features for training data, where n_samples is the number of samples in training data and len(self.regressors) is the number of regressors. refit : bool (default: True) Clones the regressors for stacking regression if True (default) or else uses the original ones, which will be refitted on the dataset upon calling the fit method. Setting refit=False is recommended if you are working with estimators that are supporting the scikit-learn fit/predict API interface but are not compatible to scikit-learn's clone function. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/regressor/StackingRegressor/ Methods fit(X, y, sample_weight=None) Learn weight coefficients from training data for each regressor. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : numpy array, shape = [n_samples] or [n_samples, n_targets] Target values. Multiple targets are supported only if self.multi_output is True. sample_weight : array-like, shape = [n_samples], optional Sample weights passed as sample_weights to each regressor in the regressors list as well as the meta_regressor. Raises error if some regressor does not support sample_weight in the fit() method. Returns self : object fit_transform(X, y=None, fit_params) Fit to data, then transform it. Fits transformer to `X` and `y` with optional parameters `fit_params` and returns a transformed version of `X`. Parameters X : array-like of shape (n_samples, n_features) Input samples. y : array-like of shape (n_samples,) or (n_samples, n_outputs), default=None Target values (None for unsupervised transformations). **fit_params : dict Additional fit parameters. Returns X_new : ndarray array of shape (n_samples, n_features_new) Transformed array. get_params(deep=True) Return estimator parameter names for GridSearch support. predict(X) Predict target values for X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns y_target : array-like, shape = [n_samples] or [n_samples, n_targets] Predicted target values. predict_meta_features(X) Get meta-features of test-data. Parameters X : numpy array, shape = [n_samples, n_features] Test vectors, where n_samples is the number of samples and n_features is the number of features. Returns meta-features : numpy array, shape = [n_samples, len(self.regressors)] meta-features for test data, where n_samples is the number of samples in test data and len(self.regressors) is the number of regressors. If self.multi_output is True, then the number of columns is len(self.regressors) * n_targets score(X, y, sample_weight=None) Return the coefficient of determination of the prediction. The coefficient of determination :math:`R^2` is defined as :math:`(1 - \\frac{u}{v})`, where :math:`u` is the residual sum of squares ((y_true - y_pred)** 2).sum() and :math: v is the total sum of squares ((y_true - y_true.mean()) ** 2).sum() . The best possible score is 1.0 and it can be negative (because the model can be arbitrarily worse). A constant model that always predicts the expected value of y , disregarding the input features, would get a :math: R^2 score of 0.0. Parameters X : array-like of shape (n_samples, n_features) Test samples. For some estimators this may be a precomputed kernel matrix or a list of generic objects instead with shape (n_samples, n_samples_fitted) , where n_samples_fitted is the number of samples used in the fitting for the estimator. y : array-like of shape (n_samples,) or (n_samples, n_outputs) True values for X . sample_weight : array-like of shape (n_samples,), default=None Sample weights. Returns score : float :math: R^2 of self.predict(X) w.r.t. y . Notes The :math: R^2 score used when calling score on a regressor uses multioutput='uniform_average' from version 0.23 to keep consistent with default value of :func: ~sklearn.metrics.r2_score . This influences the score method of all the multioutput regressors (except for :class: ~sklearn.multioutput.MultiOutputRegressor ). set_output( , transform=None)* Set output container. See :ref:`sphx_glr_auto_examples_miscellaneous_plot_set_output.py` for an example on how to use the API. Parameters transform : {\"default\", \"pandas\"}, default=None Configure output of transform and fit_transform . \"default\" : Default output format of a transformer \"pandas\" : DataFrame output None : Transform configuration is unchanged Returns self : estimator instance Estimator instance. set_params( params) Set the parameters of this estimator. Valid parameter keys can be listed with ``get_params()``. Returns self Properties coef_ None intercept_ None named_regressors None","title":"Mlxtend.regressor"},{"location":"api_subpackages/mlxtend.regressor/#linearregression","text":"LinearRegression(method='direct', eta=0.01, epochs=50, minibatches=None, random_seed=None, print_progress=0) Ordinary least squares linear regression. Parameters method : string (default: 'direct') For gradient descent-based optimization, use sgd (see minibatch parameter for further options). Otherwise, if direct (default), the analytical method is used. For alternative, numerically more stable solutions, use either qr (QR decomopisition) or svd (Singular Value Decomposition). eta : float (default: 0.01) solver learning rate (between 0.0 and 1.0). Used with method = 'sgd' . (See methods parameter for details) epochs : int (default: 50) Passes over the training dataset. Prior to each epoch, the dataset is shuffled if minibatches > 1 to prevent cycles in stochastic gradient descent. Used with method = 'sgd' . (See methods parameter for details) minibatches : int (default: None) The number of minibatches for gradient-based optimization. If None: Direct method, QR, or SVD method (see method parameter for details) If 1: Gradient Descent learning If len(y): Stochastic Gradient Descent learning If 1 < minibatches < len(y): Minibatch learning random_seed : int (default: None) Set random state for shuffling and initializing the weights. Used in method = 'sgd' . (See methods parameter for details) print_progress : int (default: 0) Prints progress in fitting to stderr if method = 'sgd' . 0: No output 1: Epochs elapsed and cost 2: 1 plus time elapsed 3: 2 plus estimated time until completion Attributes w_ : 2d-array, shape={n_features, 1} Model weights after fitting. b_ : 1d-array, shape={1,} Bias unit after fitting. cost_ : list Sum of squared errors after each epoch; ignored if solver='normal equation' Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/regressor/LinearRegression/","title":"LinearRegression"},{"location":"api_subpackages/mlxtend.regressor/#methods","text":"fit(X, y, init_params=True) Learn model from training data. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values. init_params : bool (default: True) Re-initializes model parameters prior to fitting. Set False to continue training with weights from a previous model fitting. Returns self : object get_params(deep=True) Get parameters for this estimator. Parameters deep : boolean, optional If True, will return the parameters for this estimator and contained subobjects that are estimators. Returns params : mapping of string to any Parameter names mapped to their values.' adapted from https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/base.py Author: Gael Varoquaux gael.varoquaux@normalesup.org License: BSD 3 clause predict(X) Predict targets from X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns target_values : array-like, shape = [n_samples] Predicted target values. set_params( params) Set the parameters of this estimator. The method works on simple estimators as well as on nested objects (such as pipelines). The latter have parameters of the form <component>__<parameter> so that it's possible to update each component of a nested object. Returns self adapted from https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/base.py Author: Gael Varoquaux <gael.varoquaux@normalesup.org> License: BSD 3 clause","title":"Methods"},{"location":"api_subpackages/mlxtend.regressor/#stackingcvregressor","text":"StackingCVRegressor(regressors, meta_regressor, cv=5, shuffle=True, random_state=None, verbose=0, refit=True, use_features_in_secondary=False, store_train_meta_features=False, n_jobs=None, pre_dispatch='2 n_jobs', multi_output=False)* A 'Stacking Cross-Validation' regressor for scikit-learn estimators. Parameters regressors : array-like, shape = [n_regressors] A list of regressors. Invoking the fit method on the StackingCVRegressor will fit clones of these original regressors that will be stored in the class attribute self.regr_ . meta_regressor : object The meta-regressor to be fitted on the ensemble of regressor cv : int, cross-validation generator or iterable, optional (default: 5) Determines the cross-validation splitting strategy. Possible inputs for cv are: - None, to use the default 5-fold cross validation, - integer, to specify the number of folds in a KFold , - An object to be used as a cross-validation generator. - An iterable yielding train, test splits. For integer/None inputs, it will use KFold cross-validation shuffle : bool (default: True) If True, and the cv argument is integer, the training data will be shuffled at fitting stage prior to cross-validation. If the cv argument is a specific cross validation technique, this argument is omitted. random_state : int, RandomState instance or None, optional (default: None) Constrols the randomness of the cv splitter. Used when cv is integer and shuffle=True . New in v0.16.0. verbose : int, optional (default=0) Controls the verbosity of the building process. New in v0.16.0 refit : bool (default: True) Clones the regressors for stacking regression if True (default) or else uses the original ones, which will be refitted on the dataset upon calling the fit method. Setting refit=False is recommended if you are working with estimators that are supporting the scikit-learn fit/predict API interface but are not compatible to scikit-learn's clone function. use_features_in_secondary : bool (default: False) If True, the meta-regressor will be trained both on the predictions of the original regressors and the original dataset. If False, the meta-regressor will be trained only on the predictions of the original regressors. store_train_meta_features : bool (default: False) If True, the meta-features computed from the training data used for fitting the meta-regressor stored in the self.train_meta_features_ array, which can be accessed after calling fit . n_jobs : int or None, optional (default=None) The number of CPUs to use to do the computation. None means 1 unless in a :obj: joblib.parallel_backend context. -1 means using all processors. See :term: Glossary <n_jobs> for more details. New in v0.16.0. pre_dispatch : int, or string, optional Controls the number of jobs that get dispatched during parallel execution. Reducing this number can be useful to avoid an explosion of memory consumption when more jobs get dispatched than CPUs can process. This parameter can be: - None, in which case all the jobs are immediately created and spawned. Use this for lightweight and fast-running jobs, to avoid delays due to on-demand spawning of the jobs - An int, giving the exact number of total jobs that are spawned - A string, giving an expression as a function of n_jobs, as in '2*n_jobs' multi_output : bool (default: False) If True, allow multi-output targets, but forbid nan or inf values. If False, y will be checked to be a vector. (New in v0.19.0.) Attributes train_meta_features : numpy array, shape = [n_samples, n_regressors] meta-features for training data, where n_samples is the number of samples in training data and len(self.regressors) is the number of regressors. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/regressor/StackingCVRegressor/","title":"StackingCVRegressor"},{"location":"api_subpackages/mlxtend.regressor/#methods_1","text":"fit(X, y, groups=None, sample_weight=None) Fit ensemble regressors and the meta-regressor. Parameters X : numpy array, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : numpy array, shape = [n_samples] or [n_samples, n_targets] Target values. Multiple targets are supported only if self.multi_output is True. groups : numpy array/None, shape = [n_samples] The group that each sample belongs to. This is used by specific folding strategies such as GroupKFold() sample_weight : array-like, shape = [n_samples], optional Sample weights passed as sample_weights to each regressor in the regressors list as well as the meta_regressor. Raises error if some regressor does not support sample_weight in the fit() method. Returns self : object fit_transform(X, y=None, fit_params) Fit to data, then transform it. Fits transformer to `X` and `y` with optional parameters `fit_params` and returns a transformed version of `X`. Parameters X : array-like of shape (n_samples, n_features) Input samples. y : array-like of shape (n_samples,) or (n_samples, n_outputs), default=None Target values (None for unsupervised transformations). **fit_params : dict Additional fit parameters. Returns X_new : ndarray array of shape (n_samples, n_features_new) Transformed array. get_params(deep=True) Get parameters for this estimator. Parameters deep : bool, default=True If True, will return the parameters for this estimator and contained subobjects that are estimators. Returns params : dict Parameter names mapped to their values. predict(X) Predict target values for X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns y_target : array-like, shape = [n_samples] or [n_samples, n_targets] Predicted target values. predict_meta_features(X) Get meta-features of test-data. Parameters X : numpy array, shape = [n_samples, n_features] Test vectors, where n_samples is the number of samples and n_features is the number of features. Returns meta-features : numpy array, shape = [n_samples, len(self.regressors)] meta-features for test data, where n_samples is the number of samples in test data and len(self.regressors) is the number of regressors. If self.multi_output is True, then the number of columns is len(self.regressors) * n_targets. score(X, y, sample_weight=None) Return the coefficient of determination of the prediction. The coefficient of determination :math:`R^2` is defined as :math:`(1 - \\frac{u}{v})`, where :math:`u` is the residual sum of squares ((y_true - y_pred)** 2).sum() and :math: v is the total sum of squares ((y_true - y_true.mean()) ** 2).sum() . The best possible score is 1.0 and it can be negative (because the model can be arbitrarily worse). A constant model that always predicts the expected value of y , disregarding the input features, would get a :math: R^2 score of 0.0. Parameters X : array-like of shape (n_samples, n_features) Test samples. For some estimators this may be a precomputed kernel matrix or a list of generic objects instead with shape (n_samples, n_samples_fitted) , where n_samples_fitted is the number of samples used in the fitting for the estimator. y : array-like of shape (n_samples,) or (n_samples, n_outputs) True values for X . sample_weight : array-like of shape (n_samples,), default=None Sample weights. Returns score : float :math: R^2 of self.predict(X) w.r.t. y . Notes The :math: R^2 score used when calling score on a regressor uses multioutput='uniform_average' from version 0.23 to keep consistent with default value of :func: ~sklearn.metrics.r2_score . This influences the score method of all the multioutput regressors (except for :class: ~sklearn.multioutput.MultiOutputRegressor ). set_output( , transform=None)* Set output container. See :ref:`sphx_glr_auto_examples_miscellaneous_plot_set_output.py` for an example on how to use the API. Parameters transform : {\"default\", \"pandas\"}, default=None Configure output of transform and fit_transform . \"default\" : Default output format of a transformer \"pandas\" : DataFrame output None : Transform configuration is unchanged Returns self : estimator instance Estimator instance. set_params( params) Set the parameters of this estimator. Valid parameter keys can be listed with ``get_params()``. Returns self","title":"Methods"},{"location":"api_subpackages/mlxtend.regressor/#properties","text":"named_regressors Returns List of named estimator tuples, like [('svc', SVC(...))]","title":"Properties"},{"location":"api_subpackages/mlxtend.regressor/#stackingregressor","text":"StackingRegressor(regressors, meta_regressor, verbose=0, use_features_in_secondary=False, store_train_meta_features=False, refit=True, multi_output=False) A Stacking regressor for scikit-learn estimators for regression. Parameters regressors : array-like, shape = [n_regressors] A list of regressors. Invoking the fit method on the StackingRegressor will fit clones of those original regressors that will be stored in the class attribute self.regr_ . meta_regressor : object The meta-regressor to be fitted on the ensemble of regressors verbose : int, optional (default=0) Controls the verbosity of the building process. - verbose=0 (default): Prints nothing - verbose=1 : Prints the number & name of the regressor being fitted - verbose=2 : Prints info about the parameters of the regressor being fitted - verbose>2 : Changes verbose param of the underlying regressor to self.verbose - 2 use_features_in_secondary : bool (default: False) If True, the meta-regressor will be trained both on the predictions of the original regressors and the original dataset. If False, the meta-regressor will be trained only on the predictions of the original regressors. store_train_meta_features : bool (default: False) If True, the meta-features computed from the training data used for fitting the meta-regressor stored in the self.train_meta_features_ array, which can be accessed after calling fit . Attributes regr_ : list, shape=[n_regressors] Fitted regressors (clones of the original regressors) meta_regr_ : estimator Fitted meta-regressor (clone of the original meta-estimator) coef_ : array-like, shape = [n_features] Model coefficients of the fitted meta-estimator intercept_ : float Intercept of the fitted meta-estimator train_meta_features : numpy array, shape = [n_samples, len(self.regressors)] meta-features for training data, where n_samples is the number of samples in training data and len(self.regressors) is the number of regressors. refit : bool (default: True) Clones the regressors for stacking regression if True (default) or else uses the original ones, which will be refitted on the dataset upon calling the fit method. Setting refit=False is recommended if you are working with estimators that are supporting the scikit-learn fit/predict API interface but are not compatible to scikit-learn's clone function. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/regressor/StackingRegressor/","title":"StackingRegressor"},{"location":"api_subpackages/mlxtend.regressor/#methods_2","text":"fit(X, y, sample_weight=None) Learn weight coefficients from training data for each regressor. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : numpy array, shape = [n_samples] or [n_samples, n_targets] Target values. Multiple targets are supported only if self.multi_output is True. sample_weight : array-like, shape = [n_samples], optional Sample weights passed as sample_weights to each regressor in the regressors list as well as the meta_regressor. Raises error if some regressor does not support sample_weight in the fit() method. Returns self : object fit_transform(X, y=None, fit_params) Fit to data, then transform it. Fits transformer to `X` and `y` with optional parameters `fit_params` and returns a transformed version of `X`. Parameters X : array-like of shape (n_samples, n_features) Input samples. y : array-like of shape (n_samples,) or (n_samples, n_outputs), default=None Target values (None for unsupervised transformations). **fit_params : dict Additional fit parameters. Returns X_new : ndarray array of shape (n_samples, n_features_new) Transformed array. get_params(deep=True) Return estimator parameter names for GridSearch support. predict(X) Predict target values for X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns y_target : array-like, shape = [n_samples] or [n_samples, n_targets] Predicted target values. predict_meta_features(X) Get meta-features of test-data. Parameters X : numpy array, shape = [n_samples, n_features] Test vectors, where n_samples is the number of samples and n_features is the number of features. Returns meta-features : numpy array, shape = [n_samples, len(self.regressors)] meta-features for test data, where n_samples is the number of samples in test data and len(self.regressors) is the number of regressors. If self.multi_output is True, then the number of columns is len(self.regressors) * n_targets score(X, y, sample_weight=None) Return the coefficient of determination of the prediction. The coefficient of determination :math:`R^2` is defined as :math:`(1 - \\frac{u}{v})`, where :math:`u` is the residual sum of squares ((y_true - y_pred)** 2).sum() and :math: v is the total sum of squares ((y_true - y_true.mean()) ** 2).sum() . The best possible score is 1.0 and it can be negative (because the model can be arbitrarily worse). A constant model that always predicts the expected value of y , disregarding the input features, would get a :math: R^2 score of 0.0. Parameters X : array-like of shape (n_samples, n_features) Test samples. For some estimators this may be a precomputed kernel matrix or a list of generic objects instead with shape (n_samples, n_samples_fitted) , where n_samples_fitted is the number of samples used in the fitting for the estimator. y : array-like of shape (n_samples,) or (n_samples, n_outputs) True values for X . sample_weight : array-like of shape (n_samples,), default=None Sample weights. Returns score : float :math: R^2 of self.predict(X) w.r.t. y . Notes The :math: R^2 score used when calling score on a regressor uses multioutput='uniform_average' from version 0.23 to keep consistent with default value of :func: ~sklearn.metrics.r2_score . This influences the score method of all the multioutput regressors (except for :class: ~sklearn.multioutput.MultiOutputRegressor ). set_output( , transform=None)* Set output container. See :ref:`sphx_glr_auto_examples_miscellaneous_plot_set_output.py` for an example on how to use the API. Parameters transform : {\"default\", \"pandas\"}, default=None Configure output of transform and fit_transform . \"default\" : Default output format of a transformer \"pandas\" : DataFrame output None : Transform configuration is unchanged Returns self : estimator instance Estimator instance. set_params( params) Set the parameters of this estimator. Valid parameter keys can be listed with ``get_params()``. Returns self","title":"Methods"},{"location":"api_subpackages/mlxtend.regressor/#properties_1","text":"coef_ None intercept_ None named_regressors None","title":"Properties"},{"location":"api_subpackages/mlxtend.text/","text":"mlxtend version: 0.23.1 generalize_names generalize_names(name, output_sep=' ', firstname_output_letters=1) Generalize a person's first and last name. Returns a person's name in the format <last_name><separator><firstname letter(s)> (all lowercase) Parameters name : str Name of the player output_sep : str (default: ' ') String for separating last name and first name in the output. firstname_output_letters : int Number of letters in the abbreviated first name. Returns gen_name : str The generalized name. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/text/generalize_names/ generalize_names_duplcheck generalize_names_duplcheck(df, col_name) Generalizes names and removes duplicates. Applies mlxtend.text.generalize_names to a DataFrame with 1 first name letter by default and uses more first name letters if duplicates are detected. Parameters df : pandas.DataFrame DataFrame that contains a column where generalize_names should be applied. col_name : str Name of the DataFrame column where generalize_names function should be applied to. Returns df_new : str New DataFrame object where generalize_names function has been applied without duplicates. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/text/generalize_names_duplcheck/ tokenizer_emoticons tokenizer_emoticons(text) Return emoticons from text Examples >>> tokenizer_emoticons('</a>This :) is :( a test :-)!') [':)', ':(', ':-)'] For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/text/tokenizer_emoticons/ tokenizer_words_and_emoticons tokenizer_words_and_emoticons(text) Convert text to lowercase words and emoticons. Examples >>> tokenizer_words_and_emoticons('</a>This :) is :( a test :-)!') ['this', 'is', 'a', 'test', ':)', ':(', ':-)'] For more usage examples, please see https://rasbt.github.io/mlxtend/user_guide/text/tokenizer_words_and_emoticons/","title":"Mlxtend.text"},{"location":"api_subpackages/mlxtend.text/#generalize_names","text":"generalize_names(name, output_sep=' ', firstname_output_letters=1) Generalize a person's first and last name. Returns a person's name in the format <last_name><separator><firstname letter(s)> (all lowercase) Parameters name : str Name of the player output_sep : str (default: ' ') String for separating last name and first name in the output. firstname_output_letters : int Number of letters in the abbreviated first name. Returns gen_name : str The generalized name. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/text/generalize_names/","title":"generalize_names"},{"location":"api_subpackages/mlxtend.text/#generalize_names_duplcheck","text":"generalize_names_duplcheck(df, col_name) Generalizes names and removes duplicates. Applies mlxtend.text.generalize_names to a DataFrame with 1 first name letter by default and uses more first name letters if duplicates are detected. Parameters df : pandas.DataFrame DataFrame that contains a column where generalize_names should be applied. col_name : str Name of the DataFrame column where generalize_names function should be applied to. Returns df_new : str New DataFrame object where generalize_names function has been applied without duplicates. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/text/generalize_names_duplcheck/","title":"generalize_names_duplcheck"},{"location":"api_subpackages/mlxtend.text/#tokenizer_emoticons","text":"tokenizer_emoticons(text) Return emoticons from text Examples >>> tokenizer_emoticons('</a>This :) is :( a test :-)!') [':)', ':(', ':-)'] For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/text/tokenizer_emoticons/","title":"tokenizer_emoticons"},{"location":"api_subpackages/mlxtend.text/#tokenizer_words_and_emoticons","text":"tokenizer_words_and_emoticons(text) Convert text to lowercase words and emoticons. Examples >>> tokenizer_words_and_emoticons('</a>This :) is :( a test :-)!') ['this', 'is', 'a', 'test', ':)', ':(', ':-)'] For more usage examples, please see https://rasbt.github.io/mlxtend/user_guide/text/tokenizer_words_and_emoticons/","title":"tokenizer_words_and_emoticons"},{"location":"api_subpackages/mlxtend.utils/","text":"mlxtend version: 0.23.1 Counter Counter(stderr=False, start_newline=True, precision=0, name=None) Class to display the progress of for-loop iterators. Parameters stderr : bool (default: True) Prints output to sys.stderr if True; uses sys.stdout otherwise. start_newline : bool (default: True) Prepends a new line to the counter, which prevents overwriting counters if multiple counters are printed in succession. precision: int (default: 0) Sets the number of decimal places when displaying the time elapsed in seconds. name : string (default: None) Prepends the specified name before the counter to allow distinguishing between multiple counters. Attributes curr_iter : int The current iteration. start_time : float The system's time in seconds when the Counter was initialized. end_time : float The system's time in seconds when the Counter was last updated. Examples >>> cnt = Counter() >>> for i in range(20): ... # do some computation ... time.sleep(0.1) ... cnt.update() 20 iter | 2 sec >>> print('The counter was initialized.' ' %d seconds ago.' % (time.time() - cnt.start_time)) The counter was initialized 2 seconds ago >>> print('The counter was last updated' ' %d seconds ago.' % (time.time() - cnt.end_time)) The counter was last updated 0 seconds ago. For more usage examples, please see https://rasbt.github.io/mlxtend/user_guide/utils/Counter/ Methods update() Print current iteration and time elapsed. assert_raises assert_raises(exception_type, message, func, args, * kwargs) Check that an exception is raised with a specific message Parameters exception_type : exception The exception that should be raised message : str (default: None) The error message that should be raised. Ignored if False or None. func : callable The function that raises the exception *args : positional arguments to func . **kwargs : keyword arguments to func check_Xy check_Xy(X, y, y_int=True) None format_kwarg_dictionaries format_kwarg_dictionaries(default_kwargs=None, user_kwargs=None, protected_keys=None) Function to combine default and user specified kwargs dictionaries Parameters default_kwargs : dict, optional Default kwargs (default is None). user_kwargs : dict, optional User specified kwargs (default is None). protected_keys : array_like, optional Sequence of keys to be removed from the returned dictionary (default is None). Returns formatted_kwargs : dict Formatted kwargs dictionary.","title":"Mlxtend.utils"},{"location":"api_subpackages/mlxtend.utils/#counter","text":"Counter(stderr=False, start_newline=True, precision=0, name=None) Class to display the progress of for-loop iterators. Parameters stderr : bool (default: True) Prints output to sys.stderr if True; uses sys.stdout otherwise. start_newline : bool (default: True) Prepends a new line to the counter, which prevents overwriting counters if multiple counters are printed in succession. precision: int (default: 0) Sets the number of decimal places when displaying the time elapsed in seconds. name : string (default: None) Prepends the specified name before the counter to allow distinguishing between multiple counters. Attributes curr_iter : int The current iteration. start_time : float The system's time in seconds when the Counter was initialized. end_time : float The system's time in seconds when the Counter was last updated. Examples >>> cnt = Counter() >>> for i in range(20): ... # do some computation ... time.sleep(0.1) ... cnt.update() 20 iter | 2 sec >>> print('The counter was initialized.' ' %d seconds ago.' % (time.time() - cnt.start_time)) The counter was initialized 2 seconds ago >>> print('The counter was last updated' ' %d seconds ago.' % (time.time() - cnt.end_time)) The counter was last updated 0 seconds ago. For more usage examples, please see https://rasbt.github.io/mlxtend/user_guide/utils/Counter/","title":"Counter"},{"location":"api_subpackages/mlxtend.utils/#methods","text":"update() Print current iteration and time elapsed.","title":"Methods"},{"location":"api_subpackages/mlxtend.utils/#assert_raises","text":"assert_raises(exception_type, message, func, args, * kwargs) Check that an exception is raised with a specific message Parameters exception_type : exception The exception that should be raised message : str (default: None) The error message that should be raised. Ignored if False or None. func : callable The function that raises the exception *args : positional arguments to func . **kwargs : keyword arguments to func","title":"assert_raises"},{"location":"api_subpackages/mlxtend.utils/#check_xy","text":"check_Xy(X, y, y_int=True) None","title":"check_Xy"},{"location":"api_subpackages/mlxtend.utils/#format_kwarg_dictionaries","text":"format_kwarg_dictionaries(default_kwargs=None, user_kwargs=None, protected_keys=None) Function to combine default and user specified kwargs dictionaries Parameters default_kwargs : dict, optional Default kwargs (default is None). user_kwargs : dict, optional User specified kwargs (default is None). protected_keys : array_like, optional Sequence of keys to be removed from the returned dictionary (default is None). Returns formatted_kwargs : dict Formatted kwargs dictionary.","title":"format_kwarg_dictionaries"},{"location":"user_guide/classifier/Adaline/","text":"Adaline: Adaptive Linear Neuron Classifier An implementation of the ADAptive LInear NEuron, Adaline, for binary classification tasks. from mlxtend.classifier import Adaline Overview An illustration of the ADAptive LInear NEuron (Adaline) -- a single-layer artificial linear neuron with a threshold unit: The Adaline classifier is closely related to the Ordinary Least Squares (OLS) Linear Regression algorithm; in OLS regression we find the line (or hyperplane) that minimizes the vertical offsets. Or in other words, we define the best-fitting line as the line that minimizes the sum of squared errors (SSE) or mean squared error (MSE) between our target variable (y) and our predicted output over all samples i in our dataset of size n . SSE = \\sum_i (\\text{target}^{(i)} - \\text{output}^{(i)})^2 MSE = \\frac{1}{n} \\times SSE LinearRegression implements a linear regression model for performing ordinary least squares regression, and in Adaline, we add a threshold function g(\\cdot) to convert the continuous outcome to a categorical class label: $$y = g({z}) = \\begin{cases} 1 & \\text{if z $\\ge$ 0}\\\\ -1 & \\text{otherwise}. \\end{cases} $$ An Adaline model can be trained by one of the following three approaches: Normal Equations Gradient Descent Stochastic Gradient Descent Normal Equations (closed-form solution) The closed-form solution should be preferred for \"smaller\" datasets where calculating (a \"costly\") matrix inverse is not a concern. For very large datasets, or datasets where the inverse of [X^T X] may not exist (the matrix is non-invertible or singular, e.g., in case of perfect multicollinearity), the gradient descent or stochastic gradient descent approaches are to be preferred. The linear function (linear regression model) is defined as: z = w_0x_0 + w_1x_1 + ... + w_mx_m = \\sum_{j=0}^{m} w_j x_j = \\mathbf{w}^T\\mathbf{x} where y is the response variable, \\mathbf{x} is an m -dimensional sample vector, and \\mathbf{w} is the weight vector (vector of coefficients). Note that w_0 represents the y-axis intercept of the model and therefore x_0=1 . Using the closed-form solution (normal equation), we compute the weights of the model as follows: \\mathbf{w} = (\\mathbf{X}^T\\mathbf{X})^{-1}\\mathbf{X}^Ty Gradient Descent (GD) and Stochastic Gradient Descent (SGD) In the current implementation, the Adaline model is learned via Gradient Descent or Stochastic Gradient Descent. See Gradient Descent and Stochastic Gradient Descent and Deriving the Gradient Descent Rule for Linear Regression and Adaline for details. Random shuffling is implemented as: for one or more epochs randomly shuffle samples in the training set for training sample i compute gradients and perform weight updates References B. Widrow, M. E. Hoff, et al. Adaptive switching circuits . 1960. Example 1 - Closed Form Solution from mlxtend.data import iris_data from mlxtend.plotting import plot_decision_regions from mlxtend.classifier import Adaline import matplotlib.pyplot as plt # Loading Data X, y = iris_data() X = X[:, [0, 3]] # sepal length and petal width X = X[0:100] # class 0 and class 1 y = y[0:100] # class 0 and class 1 # standardize X[:,0] = (X[:,0] - X[:,0].mean()) / X[:,0].std() X[:,1] = (X[:,1] - X[:,1].mean()) / X[:,1].std() ada = Adaline(epochs=30, eta=0.01, minibatches=None, random_seed=1) ada.fit(X, y) plot_decision_regions(X, y, clf=ada) plt.title('Adaline - Closed Form') plt.show() Example 2 - Gradient Descent from mlxtend.data import iris_data from mlxtend.plotting import plot_decision_regions from mlxtend.classifier import Adaline import matplotlib.pyplot as plt # Loading Data X, y = iris_data() X = X[:, [0, 3]] # sepal length and petal width X = X[0:100] # class 0 and class 1 y = y[0:100] # class 0 and class 1 # standardize X[:,0] = (X[:,0] - X[:,0].mean()) / X[:,0].std() X[:,1] = (X[:,1] - X[:,1].mean()) / X[:,1].std() ada = Adaline(epochs=30, eta=0.01, minibatches=1, # for Gradient Descent Learning random_seed=1, print_progress=3) ada.fit(X, y) plot_decision_regions(X, y, clf=ada) plt.title('Adaline - Gradient Descent') plt.show() plt.plot(range(len(ada.cost_)), ada.cost_) plt.xlabel('Iterations') plt.ylabel('Cost') Iteration: 30/30 | Cost 3.79 | Elapsed: 0:00:00 | ETA: 0:00:00 Text(0, 0.5, 'Cost') Example 3 - Stochastic Gradient Descent from mlxtend.data import iris_data from mlxtend.plotting import plot_decision_regions from mlxtend.classifier import Adaline import matplotlib.pyplot as plt # Loading Data X, y = iris_data() X = X[:, [0, 3]] # sepal length and petal width X = X[0:100] # class 0 and class 1 y = y[0:100] # class 0 and class 1 # standardize X[:,0] = (X[:,0] - X[:,0].mean()) / X[:,0].std() X[:,1] = (X[:,1] - X[:,1].mean()) / X[:,1].std() ada = Adaline(epochs=15, eta=0.02, minibatches=len(y), # for SGD learning random_seed=1, print_progress=3) ada.fit(X, y) plot_decision_regions(X, y, clf=ada) plt.title('Adaline - Stochastic Gradient Descent') plt.show() plt.plot(range(len(ada.cost_)), ada.cost_) plt.xlabel('Iterations') plt.ylabel('Cost') plt.show() Iteration: 15/15 | Cost 3.81 | Elapsed: 0:00:00 | ETA: 0:00:00 Example 4 - Stochastic Gradient Descent with Minibatches from mlxtend.data import iris_data from mlxtend.plotting import plot_decision_regions from mlxtend.classifier import Adaline import matplotlib.pyplot as plt # Loading Data X, y = iris_data() X = X[:, [0, 3]] # sepal length and petal width X = X[0:100] # class 0 and class 1 y = y[0:100] # class 0 and class 1 # standardize X[:,0] = (X[:,0] - X[:,0].mean()) / X[:,0].std() X[:,1] = (X[:,1] - X[:,1].mean()) / X[:,1].std() ada = Adaline(epochs=15, eta=0.02, minibatches=5, # for SGD learning w. minibatch size 20 random_seed=1, print_progress=3) ada.fit(X, y) plot_decision_regions(X, y, clf=ada) plt.title('Adaline - Stochastic Gradient Descent w. Minibatches') plt.show() plt.plot(range(len(ada.cost_)), ada.cost_) plt.xlabel('Iterations') plt.ylabel('Cost') plt.show() Iteration: 15/15 | Cost 3.87 | Elapsed: 0:00:00 | ETA: 0:00:00 API Adaline(eta=0.01, epochs=50, minibatches=None, random_seed=None, print_progress=0) ADAptive LInear NEuron classifier. Note that this implementation of Adaline expects binary class labels in {0, 1}. Parameters eta : float (default: 0.01) solver rate (between 0.0 and 1.0) epochs : int (default: 50) Passes over the training dataset. Prior to each epoch, the dataset is shuffled if minibatches > 1 to prevent cycles in stochastic gradient descent. minibatches : int (default: None) The number of minibatches for gradient-based optimization. If None: Normal Equations (closed-form solution) If 1: Gradient Descent learning If len(y): Stochastic Gradient Descent (SGD) online learning If 1 < minibatches < len(y): SGD Minibatch learning random_seed : int (default: None) Set random state for shuffling and initializing the weights. print_progress : int (default: 0) Prints progress in fitting to stderr if not solver='normal equation' 0: No output 1: Epochs elapsed and cost 2: 1 plus time elapsed 3: 2 plus estimated time until completion Attributes w_ : 2d-array, shape={n_features, 1} Model weights after fitting. b_ : 1d-array, shape={1,} Bias unit after fitting. cost_ : list Sum of squared errors after each epoch. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/classifier/Adaline/ Methods fit(X, y, init_params=True) Learn model from training data. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values. init_params : bool (default: True) Re-initializes model parameters prior to fitting. Set False to continue training with weights from a previous model fitting. Returns self : object get_params(deep=True) Get parameters for this estimator. Parameters deep : boolean, optional If True, will return the parameters for this estimator and contained subobjects that are estimators. Returns params : mapping of string to any Parameter names mapped to their values.' adapted from https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/base.py Author: Gael Varoquaux gael.varoquaux@normalesup.org License: BSD 3 clause predict(X) Predict targets from X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns target_values : array-like, shape = [n_samples] Predicted target values. score(X, y) Compute the prediction accuracy Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values (true class labels). Returns acc : float The prediction accuracy as a float between 0.0 and 1.0 (perfect score). set_params( params) Set the parameters of this estimator. The method works on simple estimators as well as on nested objects (such as pipelines). The latter have parameters of the form <component>__<parameter> so that it's possible to update each component of a nested object. Returns self adapted from https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/base.py Author: Gael Varoquaux gael.varoquaux@normalesup.org License: BSD 3 clause","title":"Adaline: Adaptive Linear Neuron Classifier"},{"location":"user_guide/classifier/Adaline/#adaline-adaptive-linear-neuron-classifier","text":"An implementation of the ADAptive LInear NEuron, Adaline, for binary classification tasks. from mlxtend.classifier import Adaline","title":"Adaline: Adaptive Linear Neuron Classifier"},{"location":"user_guide/classifier/Adaline/#overview","text":"An illustration of the ADAptive LInear NEuron (Adaline) -- a single-layer artificial linear neuron with a threshold unit: The Adaline classifier is closely related to the Ordinary Least Squares (OLS) Linear Regression algorithm; in OLS regression we find the line (or hyperplane) that minimizes the vertical offsets. Or in other words, we define the best-fitting line as the line that minimizes the sum of squared errors (SSE) or mean squared error (MSE) between our target variable (y) and our predicted output over all samples i in our dataset of size n . SSE = \\sum_i (\\text{target}^{(i)} - \\text{output}^{(i)})^2 MSE = \\frac{1}{n} \\times SSE LinearRegression implements a linear regression model for performing ordinary least squares regression, and in Adaline, we add a threshold function g(\\cdot) to convert the continuous outcome to a categorical class label: $$y = g({z}) = \\begin{cases} 1 & \\text{if z $\\ge$ 0}\\\\ -1 & \\text{otherwise}. \\end{cases} $$ An Adaline model can be trained by one of the following three approaches: Normal Equations Gradient Descent Stochastic Gradient Descent","title":"Overview"},{"location":"user_guide/classifier/Adaline/#normal-equations-closed-form-solution","text":"The closed-form solution should be preferred for \"smaller\" datasets where calculating (a \"costly\") matrix inverse is not a concern. For very large datasets, or datasets where the inverse of [X^T X] may not exist (the matrix is non-invertible or singular, e.g., in case of perfect multicollinearity), the gradient descent or stochastic gradient descent approaches are to be preferred. The linear function (linear regression model) is defined as: z = w_0x_0 + w_1x_1 + ... + w_mx_m = \\sum_{j=0}^{m} w_j x_j = \\mathbf{w}^T\\mathbf{x} where y is the response variable, \\mathbf{x} is an m -dimensional sample vector, and \\mathbf{w} is the weight vector (vector of coefficients). Note that w_0 represents the y-axis intercept of the model and therefore x_0=1 . Using the closed-form solution (normal equation), we compute the weights of the model as follows: \\mathbf{w} = (\\mathbf{X}^T\\mathbf{X})^{-1}\\mathbf{X}^Ty","title":"Normal Equations (closed-form solution)"},{"location":"user_guide/classifier/Adaline/#gradient-descent-gd-and-stochastic-gradient-descent-sgd","text":"In the current implementation, the Adaline model is learned via Gradient Descent or Stochastic Gradient Descent. See Gradient Descent and Stochastic Gradient Descent and Deriving the Gradient Descent Rule for Linear Regression and Adaline for details. Random shuffling is implemented as: for one or more epochs randomly shuffle samples in the training set for training sample i compute gradients and perform weight updates","title":"Gradient Descent (GD)  and Stochastic Gradient Descent (SGD)"},{"location":"user_guide/classifier/Adaline/#references","text":"B. Widrow, M. E. Hoff, et al. Adaptive switching circuits . 1960.","title":"References"},{"location":"user_guide/classifier/Adaline/#example-1-closed-form-solution","text":"from mlxtend.data import iris_data from mlxtend.plotting import plot_decision_regions from mlxtend.classifier import Adaline import matplotlib.pyplot as plt # Loading Data X, y = iris_data() X = X[:, [0, 3]] # sepal length and petal width X = X[0:100] # class 0 and class 1 y = y[0:100] # class 0 and class 1 # standardize X[:,0] = (X[:,0] - X[:,0].mean()) / X[:,0].std() X[:,1] = (X[:,1] - X[:,1].mean()) / X[:,1].std() ada = Adaline(epochs=30, eta=0.01, minibatches=None, random_seed=1) ada.fit(X, y) plot_decision_regions(X, y, clf=ada) plt.title('Adaline - Closed Form') plt.show()","title":"Example 1 - Closed Form Solution"},{"location":"user_guide/classifier/Adaline/#example-2-gradient-descent","text":"from mlxtend.data import iris_data from mlxtend.plotting import plot_decision_regions from mlxtend.classifier import Adaline import matplotlib.pyplot as plt # Loading Data X, y = iris_data() X = X[:, [0, 3]] # sepal length and petal width X = X[0:100] # class 0 and class 1 y = y[0:100] # class 0 and class 1 # standardize X[:,0] = (X[:,0] - X[:,0].mean()) / X[:,0].std() X[:,1] = (X[:,1] - X[:,1].mean()) / X[:,1].std() ada = Adaline(epochs=30, eta=0.01, minibatches=1, # for Gradient Descent Learning random_seed=1, print_progress=3) ada.fit(X, y) plot_decision_regions(X, y, clf=ada) plt.title('Adaline - Gradient Descent') plt.show() plt.plot(range(len(ada.cost_)), ada.cost_) plt.xlabel('Iterations') plt.ylabel('Cost') Iteration: 30/30 | Cost 3.79 | Elapsed: 0:00:00 | ETA: 0:00:00 Text(0, 0.5, 'Cost')","title":"Example 2 - Gradient Descent"},{"location":"user_guide/classifier/Adaline/#example-3-stochastic-gradient-descent","text":"from mlxtend.data import iris_data from mlxtend.plotting import plot_decision_regions from mlxtend.classifier import Adaline import matplotlib.pyplot as plt # Loading Data X, y = iris_data() X = X[:, [0, 3]] # sepal length and petal width X = X[0:100] # class 0 and class 1 y = y[0:100] # class 0 and class 1 # standardize X[:,0] = (X[:,0] - X[:,0].mean()) / X[:,0].std() X[:,1] = (X[:,1] - X[:,1].mean()) / X[:,1].std() ada = Adaline(epochs=15, eta=0.02, minibatches=len(y), # for SGD learning random_seed=1, print_progress=3) ada.fit(X, y) plot_decision_regions(X, y, clf=ada) plt.title('Adaline - Stochastic Gradient Descent') plt.show() plt.plot(range(len(ada.cost_)), ada.cost_) plt.xlabel('Iterations') plt.ylabel('Cost') plt.show() Iteration: 15/15 | Cost 3.81 | Elapsed: 0:00:00 | ETA: 0:00:00","title":"Example 3 - Stochastic Gradient Descent"},{"location":"user_guide/classifier/Adaline/#example-4-stochastic-gradient-descent-with-minibatches","text":"from mlxtend.data import iris_data from mlxtend.plotting import plot_decision_regions from mlxtend.classifier import Adaline import matplotlib.pyplot as plt # Loading Data X, y = iris_data() X = X[:, [0, 3]] # sepal length and petal width X = X[0:100] # class 0 and class 1 y = y[0:100] # class 0 and class 1 # standardize X[:,0] = (X[:,0] - X[:,0].mean()) / X[:,0].std() X[:,1] = (X[:,1] - X[:,1].mean()) / X[:,1].std() ada = Adaline(epochs=15, eta=0.02, minibatches=5, # for SGD learning w. minibatch size 20 random_seed=1, print_progress=3) ada.fit(X, y) plot_decision_regions(X, y, clf=ada) plt.title('Adaline - Stochastic Gradient Descent w. Minibatches') plt.show() plt.plot(range(len(ada.cost_)), ada.cost_) plt.xlabel('Iterations') plt.ylabel('Cost') plt.show() Iteration: 15/15 | Cost 3.87 | Elapsed: 0:00:00 | ETA: 0:00:00","title":"Example 4 - Stochastic Gradient Descent with Minibatches"},{"location":"user_guide/classifier/Adaline/#api","text":"Adaline(eta=0.01, epochs=50, minibatches=None, random_seed=None, print_progress=0) ADAptive LInear NEuron classifier. Note that this implementation of Adaline expects binary class labels in {0, 1}. Parameters eta : float (default: 0.01) solver rate (between 0.0 and 1.0) epochs : int (default: 50) Passes over the training dataset. Prior to each epoch, the dataset is shuffled if minibatches > 1 to prevent cycles in stochastic gradient descent. minibatches : int (default: None) The number of minibatches for gradient-based optimization. If None: Normal Equations (closed-form solution) If 1: Gradient Descent learning If len(y): Stochastic Gradient Descent (SGD) online learning If 1 < minibatches < len(y): SGD Minibatch learning random_seed : int (default: None) Set random state for shuffling and initializing the weights. print_progress : int (default: 0) Prints progress in fitting to stderr if not solver='normal equation' 0: No output 1: Epochs elapsed and cost 2: 1 plus time elapsed 3: 2 plus estimated time until completion Attributes w_ : 2d-array, shape={n_features, 1} Model weights after fitting. b_ : 1d-array, shape={1,} Bias unit after fitting. cost_ : list Sum of squared errors after each epoch. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/classifier/Adaline/","title":"API"},{"location":"user_guide/classifier/Adaline/#methods","text":"fit(X, y, init_params=True) Learn model from training data. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values. init_params : bool (default: True) Re-initializes model parameters prior to fitting. Set False to continue training with weights from a previous model fitting. Returns self : object get_params(deep=True) Get parameters for this estimator. Parameters deep : boolean, optional If True, will return the parameters for this estimator and contained subobjects that are estimators. Returns params : mapping of string to any Parameter names mapped to their values.' adapted from https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/base.py Author: Gael Varoquaux gael.varoquaux@normalesup.org License: BSD 3 clause predict(X) Predict targets from X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns target_values : array-like, shape = [n_samples] Predicted target values. score(X, y) Compute the prediction accuracy Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values (true class labels). Returns acc : float The prediction accuracy as a float between 0.0 and 1.0 (perfect score). set_params( params) Set the parameters of this estimator. The method works on simple estimators as well as on nested objects (such as pipelines). The latter have parameters of the form <component>__<parameter> so that it's possible to update each component of a nested object. Returns self adapted from https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/base.py Author: Gael Varoquaux gael.varoquaux@normalesup.org License: BSD 3 clause","title":"Methods"},{"location":"user_guide/classifier/EnsembleVoteClassifier/","text":"EnsembleVoteClassifier: A majority voting classifier Implementation of a majority voting EnsembleVoteClassifier for classification. from mlxtend.classifier import EnsembleVoteClassifier Overview The EnsembleVoteClassifier is a meta-classifier for combining similar or conceptually different machine learning classifiers for classification via majority or plurality voting. (For simplicity, we will refer to both majority and plurality voting as majority voting.) The EnsembleVoteClassifier implements \"hard\" and \"soft\" voting. In hard voting, we predict the final class label as the class label that has been predicted most frequently by the classification models. In soft voting, we predict the class labels by averaging the class-probabilities (only recommended if the classifiers are well-calibrated). Note If you are interested in using the EnsembleVoteClassifier , please note that it is now also available through scikit learn (>0.17) as VotingClassifier . Majority Voting / Hard Voting Hard voting is the simplest case of majority voting. Here, we predict the class label \\hat{y} via majority (plurality) voting of each classifier C_j : \\hat{y}=mode\\{C_1(\\mathbf{x}), C_2(\\mathbf{x}), ..., C_m(\\mathbf{x})\\} Assuming that we combine three classifiers that classify a training sample as follows: classifier 1 -> class 0 classifier 2 -> class 0 classifier 3 -> class 1 \\hat{y}=mode\\{0, 0, 1\\} = 0 Via majority vote, we would classify the sample as \"class 0.\" Weighted Majority Vote In addition to the simple majority vote (hard voting) as described in the previous section, we can compute a weighted majority vote by associating a weight w_j with classifier C_j : \\hat{y} = \\arg \\max_i \\sum^{m}_{j=1} w_j \\chi_A \\big(C_j(\\mathbf{x})=i\\big), where \\chi_A is the characteristic function [C_j(\\mathbf{x}) = i \\; \\in A] , and A is the set of unique class labels. Continuing with the example from the previous section classifier 1 -> class 0 classifier 2 -> class 0 classifier 3 -> class 1 assigning the weights {0.2, 0.2, 0.6} would yield a prediction \\hat{y} = 1 : \\arg \\max_i [0.2 \\times i_0 + 0.2 \\times i_0 + 0.6 \\times i_1] = 1 Soft Voting In soft voting, we predict the class labels based on the predicted probabilities p for classifier -- this approach is only recommended if the classifiers are well-calibrated. \\hat{y} = \\arg \\max_i \\sum^{m}_{j=1} w_j p_{ij}, where w_j is the weight that can be assigned to the j th classifier. Assuming the example in the previous section was a binary classification task with class labels i \\in \\{0, 1\\} , our ensemble could make the following prediction: C_1(\\mathbf{x}) \\rightarrow [0.9, 0.1] C_2(\\mathbf{x}) \\rightarrow [0.8, 0.2] C_3(\\mathbf{x}) \\rightarrow [0.4, 0.6] Using uniform weights, we compute the average probabilities: p(i_0 \\mid \\mathbf{x}) = \\frac{0.9 + 0.8 + 0.4}{3} = 0.7 \\\\\\\\ p(i_1 \\mid \\mathbf{x}) = \\frac{0.1 + 0.2 + 0.6}{3} = 0.3 \\hat{y} = \\arg \\max_i \\big[p(i_0 \\mid \\mathbf{x}), p(i_1 \\mid \\mathbf{x}) \\big] = 0 However, assigning the weights {0.1, 0.1, 0.8} would yield a prediction \\hat{y} = 1 : p(i_0 \\mid \\mathbf{x}) = {0.1 \\times 0.9 + 0.1 \\times 0.8 + 0.8 \\times 0.4} = 0.49 \\\\\\\\ p(i_1 \\mid \\mathbf{x}) = {0.1 \\times 0.1 + 0.2 \\times 0.1 + 0.8 \\times 0.6} = 0.51 \\hat{y} = \\arg \\max_i \\big[p(i_0 \\mid \\mathbf{x}), p(i_1 \\mid \\mathbf{x}) \\big] = 1 References [1] S. Raschka. Python Machine Learning . Packt Publishing Ltd., 2015. Example 1 - Classifying Iris Flowers Using Different Classification Models from sklearn import datasets iris = datasets.load_iris() X, y = iris.data[:, 1:3], iris.target from sklearn import model_selection from sklearn.linear_model import LogisticRegression from sklearn.naive_bayes import GaussianNB from sklearn.ensemble import RandomForestClassifier import numpy as np clf1 = LogisticRegression(random_state=1) clf2 = RandomForestClassifier(random_state=1) clf3 = GaussianNB() print('5-fold cross validation:\\n') labels = ['Logistic Regression', 'Random Forest', 'Naive Bayes'] for clf, label in zip([clf1, clf2, clf3], labels): scores = model_selection.cross_val_score(clf, X, y, cv=5, scoring='accuracy') print(\"Accuracy: %0.2f (+/- %0.2f) [%s]\" % (scores.mean(), scores.std(), label)) 5-fold cross validation: Accuracy: 0.95 (+/- 0.04) [Logistic Regression] Accuracy: 0.94 (+/- 0.04) [Random Forest] Accuracy: 0.91 (+/- 0.04) [Naive Bayes] from mlxtend.classifier import EnsembleVoteClassifier eclf = EnsembleVoteClassifier(clfs=[clf1, clf2, clf3], weights=[1,1,1]) labels = ['Logistic Regression', 'Random Forest', 'Naive Bayes', 'Ensemble'] for clf, label in zip([clf1, clf2, clf3, eclf], labels): scores = model_selection.cross_val_score(clf, X, y, cv=5, scoring='accuracy') print(\"Accuracy: %0.2f (+/- %0.2f) [%s]\" % (scores.mean(), scores.std(), label)) Accuracy: 0.95 (+/- 0.04) [Logistic Regression] Accuracy: 0.94 (+/- 0.04) [Random Forest] Accuracy: 0.91 (+/- 0.04) [Naive Bayes] Accuracy: 0.95 (+/- 0.04) [Ensemble] Plotting Decision Regions import matplotlib.pyplot as plt from mlxtend.plotting import plot_decision_regions import matplotlib.gridspec as gridspec import itertools gs = gridspec.GridSpec(2, 2) fig = plt.figure(figsize=(10,8)) labels = ['Logistic Regression', 'Random Forest', 'Naive Bayes', 'Ensemble'] for clf, lab, grd in zip([clf1, clf2, clf3, eclf], labels, itertools.product([0, 1], repeat=2)): clf.fit(X, y) ax = plt.subplot(gs[grd[0], grd[1]]) fig = plot_decision_regions(X=X, y=y, clf=clf) plt.title(lab) Example 2 - Grid Search from sklearn import datasets iris = datasets.load_iris() X, y = iris.data[:, 1:3], iris.target %%capture --no-display from sklearn.model_selection import GridSearchCV from sklearn.linear_model import LogisticRegression from sklearn.naive_bayes import GaussianNB from sklearn.ensemble import RandomForestClassifier from mlxtend.classifier import EnsembleVoteClassifier clf1 = LogisticRegression(random_state=1) clf2 = RandomForestClassifier(random_state=1) clf3 = GaussianNB() eclf = EnsembleVoteClassifier(clfs=[clf1, clf2, clf3], voting='soft') params = {'logisticregression__C': [1.0, 100.0], 'randomforestclassifier__n_estimators': [20, 200],} grid = GridSearchCV(estimator=eclf, param_grid=params, cv=5) grid.fit(iris.data, iris.target) cv_keys = ('mean_test_score', 'std_test_score', 'params') for r, _ in enumerate(grid.cv_results_['mean_test_score']): print(\"%0.3f +/- %0.2f %r\" % (grid.cv_results_[cv_keys[0]][r], grid.cv_results_[cv_keys[1]][r] / 2.0, grid.cv_results_[cv_keys[2]][r])) Note : If the EnsembleClassifier is initialized with multiple similar estimator objects, the estimator names are modified with consecutive integer indices, for example: %%capture --no-display clf1 = LogisticRegression(random_state=1) clf2 = RandomForestClassifier(random_state=1) eclf = EnsembleVoteClassifier(clfs=[clf1, clf1, clf2], voting='soft') params = {'logisticregression-1__C': [1.0, 100.0], 'logisticregression-2__C': [1.0, 100.0], 'randomforestclassifier__n_estimators': [20, 200],} grid = GridSearchCV(estimator=eclf, param_grid=params, cv=5) grid = grid.fit(iris.data, iris.target) Note The EnsembleVoteClass also enables grid search over the clfs argument. However, due to the current implementation of GridSearchCV in scikit-learn, it is not possible to search over both, differenct classifiers and classifier parameters at the same time. For instance, while the following parameter dictionary works params = {'randomforestclassifier__n_estimators': [1, 100], 'clfs': [(clf1, clf1, clf1), (clf2, clf3)]} it will use the instance settings of clf1 , clf2 , and clf3 and not overwrite it with the 'n_estimators' settings from 'randomforestclassifier__n_estimators': [1, 100] . Example 3 - Majority voting with classifiers trained on different feature subsets Feature selection algorithms implemented in scikit-learn as well as the SequentialFeatureSelector implement a transform method that passes the reduced feature subset to the next item in a Pipeline . For example, the method def transform(self, X): return X[:, self.k_feature_idx_] returns the best feature columns, k_feature_idx_ , given a dataset X. Thus, we simply need to construct a Pipeline consisting of the feature selector and the classifier in order to select different feature subsets for different algorithms. During fitting , the optimal feature subsets are automatically determined via the GridSearchCV object, and by calling predict , the fitted feature selector in the pipeline only passes these columns along, which resulted in the best performance for the respective classifier. %%capture --no-display from sklearn import datasets iris = datasets.load_iris() X, y = iris.data[:, :], iris.target from sklearn.model_selection import GridSearchCV from sklearn.linear_model import LogisticRegression from sklearn.naive_bayes import GaussianNB from sklearn.ensemble import RandomForestClassifier from mlxtend.classifier import EnsembleVoteClassifier from sklearn.pipeline import Pipeline from mlxtend.feature_selection import SequentialFeatureSelector clf1 = LogisticRegression(random_state=1) clf2 = RandomForestClassifier(random_state=1) clf3 = GaussianNB() # Creating a feature-selection-classifier pipeline sfs1 = SequentialFeatureSelector(clf1, k_features=4, forward=True, floating=False, scoring='accuracy', verbose=0, cv=0) clf1_pipe = Pipeline([('sfs', sfs1), ('logreg', clf1)]) eclf = EnsembleVoteClassifier(clfs=[clf1_pipe, clf2, clf3], voting='soft') params = {'pipeline__sfs__k_features': [1, 2, 3], 'pipeline__logreg__C': [1.0, 100.0], 'randomforestclassifier__n_estimators': [20, 200]} grid = GridSearchCV(estimator=eclf, param_grid=params, cv=5) grid.fit(iris.data, iris.target) cv_keys = ('mean_test_score', 'std_test_score', 'params') for r, _ in enumerate(grid.cv_results_['mean_test_score']): print(\"%0.3f +/- %0.2f %r\" % (grid.cv_results_[cv_keys[0]][r], grid.cv_results_[cv_keys[1]][r] / 2.0, grid.cv_results_[cv_keys[2]][r])) The best parameters determined via GridSearch are: grid.best_params_ {'pipeline__logreg__C': 1.0, 'pipeline__sfs__k_features': 2, 'randomforestclassifier__n_estimators': 200} Now, we assign these parameters to the ensemble voting classifier, fit the models on the complete training set, and perform a prediction on 3 samples from the Iris dataset. eclf = eclf.set_params(**grid.best_params_) eclf.fit(X, y).predict(X[[1, 51, 149]]) array([0, 1, 2]) Manual Approach Alternatively, we can select different columns \"manually\" using the ColumnSelector object. In this example, we select only the first (sepal length) and third (petal length) column for the logistic regression classifier ( clf1 ). from mlxtend.feature_selection import ColumnSelector col_sel = ColumnSelector(cols=[0, 2]) clf1_pipe = Pipeline([('sel', col_sel), ('logreg', clf1)]) eclf = EnsembleVoteClassifier(clfs=[clf1_pipe, clf2, clf3], voting='soft') eclf.fit(X, y).predict(X[[1, 51, 149]]) array([0, 1, 2]) Furthermore, we can fit the SequentialFeatureSelector separately, outside the grid search hyperparameter optimization pipeline. Here, we determine the best features first, and then we construct a pipeline using these \"fixed,\" best features as seed for the ColumnSelector : sfs1 = SequentialFeatureSelector(clf1, k_features=2, forward=True, floating=False, scoring='accuracy', verbose=1, cv=0) sfs1.fit(X, y) print('Best features', sfs1.k_feature_idx_) col_sel = ColumnSelector(cols=sfs1.k_feature_idx_) clf1_pipe = Pipeline([('sel', col_sel), ('logreg', clf1)]) [Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers. [Parallel(n_jobs=1)]: Done 4 out of 4 | elapsed: 0.0s finished Features: 1/2[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers. [Parallel(n_jobs=1)]: Done 3 out of 3 | elapsed: 0.0s finished Features: 2/2 Best features (2, 3) eclf = EnsembleVoteClassifier(clfs=[clf1_pipe, clf2, clf3], voting='soft') eclf.fit(X, y).predict(X[[1, 51, 149]]) array([0, 1, 2]) Example 5 - Using Pre-fitted Classifiers from sklearn import datasets iris = datasets.load_iris() X, y = iris.data[:, 1:3], iris.target Assume that we previously fitted our classifiers: from sklearn import model_selection from sklearn.linear_model import LogisticRegression from sklearn.naive_bayes import GaussianNB from sklearn.ensemble import RandomForestClassifier import numpy as np clf1 = LogisticRegression(random_state=1) clf2 = RandomForestClassifier(random_state=1) clf3 = GaussianNB() for clf in (clf1, clf2, clf3): clf.fit(X, y) By setting fit_base_estimators=False , it will enforce use_clones to be False and the EnsembleVoteClassifier will not re-fit these classifers to save computational time: from mlxtend.classifier import EnsembleVoteClassifier import copy eclf = EnsembleVoteClassifier(clfs=[clf1, clf2, clf3], weights=[1,1,1], fit_base_estimators=False) labels = ['Logistic Regression', 'Random Forest', 'Naive Bayes', 'Ensemble'] eclf.fit(X, y) print('accuracy:', np.mean(y == eclf.predict(X))) accuracy: 0.96 /Users/sebastian/miniforge3/lib/python3.9/site-packages/mlxtend/classifier/ensemble_vote.py:166: UserWarning: fit_base_estimators=False enforces use_clones to be `False` warnings.warn(\"fit_base_estimators=False \" However, please note that fit_base_estimators=False is incompatible to any form of cross-validation that is done in e.g., model_selection.cross_val_score or model_selection.GridSearchCV , etc., since it would require the classifiers to be refit to the training folds. Thus, only use fit_base_estimators=False if you want to make a prediction directly without cross-validation. Example 6 - Ensembles of Classifiers that Operate on Different Feature Subsets If desired, the different classifiers can be fit to different subsets of features in the training dataset. The following example illustrates how this can be done on a technical level using scikit-learn pipelines and the ColumnSelector : from sklearn.datasets import load_iris from mlxtend.classifier import EnsembleVoteClassifier from mlxtend.feature_selection import ColumnSelector from sklearn.pipeline import make_pipeline from sklearn.linear_model import LogisticRegression iris = load_iris() X = iris.data y = iris.target pipe1 = make_pipeline(ColumnSelector(cols=(0, 2)), LogisticRegression()) pipe2 = make_pipeline(ColumnSelector(cols=(1, 2, 3)), LogisticRegression()) eclf = EnsembleVoteClassifier(clfs=[pipe1, pipe2]) eclf.fit(X, y) EnsembleVoteClassifier(clfs=[Pipeline(steps=[('columnselector', ColumnSelector(cols=(0, 2))), ('logisticregression', LogisticRegression())]), Pipeline(steps=[('columnselector', ColumnSelector(cols=(1, 2, 3))), ('logisticregression', LogisticRegression())])]) Example 7 - A Note about Scikit-Learn SVMs and Soft Voting This section provides some additional technical insights in how probabilities are used when voting='soft' . Note that scikit-learn estimates the probabilities for SVMs (more info here: https://scikit-learn.org/stable/modules/svm.html#scores-probabilities) in a way that these may not be consistent with the class labels that the SVM predicts. This is an extreme example, but let's say we have a dataset with 3 class labels, 0, 1, and 2. For a given training example, the SVM classifier may predict class 2. However, the class-membership probabilities may look as follows: class 0: 99% class 1: 0.5% class 2: 0.5% A practical example of this scenario is shown below: import numpy as np from mlxtend.classifier import EnsembleVoteClassifier from sklearn.svm import SVC from sklearn.datasets import load_iris iris = load_iris() X, y = iris.data, iris.target clf2 = SVC(probability=True, random_state=4) clf2.fit(X, y) eclf = EnsembleVoteClassifier(clfs=[clf2], voting='soft', fit_base_estimators=False) eclf.fit(X, y) for svm_class, e_class, svm_prob, e_prob, in zip(clf2.predict(X), eclf.predict(X), clf2.predict_proba(X), eclf.predict_proba(X)): if svm_class != e_class: print('============') print('Probas from SVM :', svm_prob) print('Class from SVM :', svm_class) print('Probas from SVM in Ensemble:', e_prob) print('Class from SVM in Ensemble :', e_class) print('============') ============ Probas from SVM : [0.00921708 0.49415165 0.49663127] Class from SVM : 1 Probas from SVM in Ensemble: [0.00921708 0.49415165 0.49663127] Class from SVM in Ensemble : 2 ============ /Users/sebastian/miniforge3/lib/python3.9/site-packages/mlxtend/classifier/ensemble_vote.py:166: UserWarning: fit_base_estimators=False enforces use_clones to be `False` warnings.warn(\"fit_base_estimators=False \" Example 7 - A Note about Scikit-Learn SVMs and Soft Voting Based on the probabilities, we would expect the SVM to predict class 2, because it has the highest probability. Since the EnsembleVoteClassifier uses the argmax function internally if voting='soft' , it would indeed predict class 2 in this case even if the ensemble consists of only one SVM model. Note that in practice, this minor technical detail does not need to concern you, but it is useful to keep it in mind in case you are wondering about results from a 1-model SVM ensemble compared to that SVM alone -- this is not a bug. Example 8 - Optimizing Ensemble Weights with Nelder-Mead In this section, we will see how we can use a heuristic search method like Nelder-Mead for optimizing the ensemble weights. Suppose we have the following example scenario where we fit 3 individual classifiers on different subsets of the training dataset: from mlxtend.classifier import EnsembleVoteClassifier from mlxtend.data import mnist_data from sklearn.linear_model import LogisticRegression from sklearn.model_selection import train_test_split from sklearn.naive_bayes import GaussianNB from sklearn.tree import DecisionTreeClassifier X, y = mnist_data() X_train, X_val, y_train, y_val = train_test_split( X, y, test_size=0.5, shuffle=True, random_state=1 ) clf1 = GaussianNB() clf2 = LogisticRegression(random_state=123, solver='newton-cg') clf3 = DecisionTreeClassifier(random_state=123, max_depth=2) clf1.fit(X_train[500:1000], y_train[500:1000]) clf2.fit(X_train[750:1250], y_train[750:1250]) clf3.fit(X_train[1250:2000], y_train[1250:2000]); Then, we construct the an ensemble classifier from these 3 classifiers where each classifier contributes equally with weight 1: eclf = EnsembleVoteClassifier( clfs=(clf1, clf2, clf3), voting=\"soft\", # the same would also work with \"hard\" voting weights=(1, 1, 1), use_clones=False, fit_base_estimators=False, ) eclf.fit(X_train, y_train) eclf.score(X_val, y_val) /Users/sebastian/miniforge3/lib/python3.9/site-packages/mlxtend/classifier/ensemble_vote.py:166: UserWarning: fit_base_estimators=False enforces use_clones to be `False` warnings.warn(\"fit_base_estimators=False \" 0.8012 We see that we reach 80% accuracy on the validation set. Can we do better? Maybe they indvidual classifiers shouldn't be contributing equally. Perhaps, we can use an optimization algorothm from scipy.optimize to find a better relative weighting of these individual classifiers. Let's set up an objective function that we want to minimize via SciPy's minimize : from scipy.optimize import minimize def function_to_minimize(weights, fitted_clfs): w1, w2 = weights # these are the new weights! newclf = EnsembleVoteClassifier( voting=\"soft\", use_clones=False, fit_base_estimators=False, clfs=fitted_clfs, weights=(w1, w2, 1.), # use the new weights ) newclf.fit(X_train, y_train) score = newclf.score(X_val, y_val) # change accuracy to error so that smaller is better score_to_minimize = 1 - score return score_to_minimize Note a few things: We only optimize 2 out of the 3 classifier weigths. That's because the weighting is relative to each other, and it would be overkill (and too many degrees of freedom) to also optimize weight 3. We set use_clones=False & fit_base_estimators=False as before, this is to make sure that we use the prefit classifiers in the ensemble classifier. Instead of optimizing the accuracy, we optimize the the classification error, score_to_minimize = 1 - score . That's because we use the minimize function where lower means better. Next, let's choose some initial weight values and run the optimization. Via the bounds we specify the range (lower and upper value) for each weight so that the search doesn't go crazy: %%capture --no-display init_weights = [1., 1.] results = minimize( function_to_minimize, init_weights, args=((clf1, clf2, clf3),), bounds=((0, 5), (0, 5)), method=\"nelder-mead\", ) Let's look at the results! print(results) final_simplex: (array([[0.575 , 1.40625 ], [0.57500153, 1.40622215], [0.57508965, 1.40617647]]), array([0.1324, 0.1324, 0.1324])) fun: 0.13239999999999996 message: 'Optimization terminated successfully.' nfev: 60 nit: 21 status: 0 success: True x: array([0.575 , 1.40625]) It looks like the search was successful and returned the following weights: solution = results[\"x\"] print(solution) [0.575 1.40625] Let's use these new weights in our ensemble classifier: eclf = EnsembleVoteClassifier( clfs=(clf1, clf2, clf3), voting=\"soft\", weights=(solution[0], solution[1], 1), use_clones=False, fit_base_estimators=False, ) eclf.fit(X_train, y_train) eclf.score(X_val, y_val) /Users/sebastian/miniforge3/lib/python3.9/site-packages/mlxtend/classifier/ensemble_vote.py:166: UserWarning: fit_base_estimators=False enforces use_clones to be `False` warnings.warn(\"fit_base_estimators=False \" 0.8676 As we can see, the results on the validation set (0.8676) improved compared to the original ones (0.8012). Yay! API EnsembleVoteClassifier(clfs, voting='hard', weights=None, verbose=0, use_clones=True, fit_base_estimators=True) Soft Voting/Majority Rule classifier for scikit-learn estimators. Parameters clfs : array-like, shape = [n_classifiers] A list of classifiers. Invoking the fit method on the VotingClassifier will fit clones of those original classifiers be stored in the class attribute if use_clones=True (default) and fit_base_estimators=True (default). voting : str, {'hard', 'soft'} (default='hard') If 'hard', uses predicted class labels for majority rule voting. Else if 'soft', predicts the class label based on the argmax of the sums of the predicted probalities, which is recommended for an ensemble of well-calibrated classifiers. weights : array-like, shape = [n_classifiers], optional (default= None ) Sequence of weights ( float or int ) to weight the occurances of predicted class labels ( hard voting) or class probabilities before averaging ( soft voting). Uses uniform weights if None . verbose : int, optional (default=0) Controls the verbosity of the building process. - verbose=0 (default): Prints nothing - verbose=1 : Prints the number & name of the clf being fitted - verbose=2 : Prints info about the parameters of the clf being fitted - verbose>2 : Changes verbose param of the underlying clf to self.verbose - 2 use_clones : bool (default: True) Clones the classifiers for stacking classification if True (default) or else uses the original ones, which will be refitted on the dataset upon calling the fit method. Hence, if use_clones=True, the original input classifiers will remain unmodified upon using the StackingClassifier's fit method. Setting use_clones=False is recommended if you are working with estimators that are supporting the scikit-learn fit/predict API interface but are not compatible to scikit-learn's clone function. fit_base_estimators : bool (default: True) Refits classifiers in clfs if True; uses references to the clfs , otherwise (assumes that the classifiers were already fit). Note: fit_base_estimators=False will enforce use_clones to be False, and is incompatible to most scikit-learn wrappers! For instance, if any form of cross-validation is performed this would require the re-fitting classifiers to training folds, which would raise a NotFitterError if fit_base_estimators=False. (New in mlxtend v0.6.) Attributes classes_ : array-like, shape = [n_predictions] clf : array-like, shape = [n_predictions] The input classifiers; may be overwritten if use_clones=False clf_ : array-like, shape = [n_predictions] Fitted input classifiers; clones if use_clones=True Examples ``` >>> import numpy as np >>> from sklearn.linear_model import LogisticRegression >>> from sklearn.naive_bayes import GaussianNB >>> from sklearn.ensemble import RandomForestClassifier >>> from mlxtend.sklearn import EnsembleVoteClassifier >>> clf1 = LogisticRegression(random_seed=1) >>> clf2 = RandomForestClassifier(random_seed=1) >>> clf3 = GaussianNB() >>> X = np.array([[-1, -1], [-2, -1], [-3, -2], [1, 1], [2, 1], [3, 2]]) >>> y = np.array([1, 1, 1, 2, 2, 2]) >>> eclf1 = EnsembleVoteClassifier(clfs=[clf1, clf2, clf3], ... voting='hard', verbose=1) >>> eclf1 = eclf1.fit(X, y) >>> print(eclf1.predict(X)) [1 1 1 2 2 2] >>> eclf2 = EnsembleVoteClassifier(clfs=[clf1, clf2, clf3], voting='soft') >>> eclf2 = eclf2.fit(X, y) >>> print(eclf2.predict(X)) [1 1 1 2 2 2] >>> eclf3 = EnsembleVoteClassifier(clfs=[clf1, clf2, clf3], ... voting='soft', weights=[2,1,1]) >>> eclf3 = eclf3.fit(X, y) >>> print(eclf3.predict(X)) [1 1 1 2 2 2] >>> For more usage examples, please see https://rasbt.github.io/mlxtend/user_guide/classifier/EnsembleVoteClassifier/ ``` Methods fit(X, y, sample_weight=None) Learn weight coefficients from training data for each classifier. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values. sample_weight : array-like, shape = [n_samples], optional Sample weights passed as sample_weights to each regressor in the regressors list as well as the meta_regressor. Raises error if some regressor does not support sample_weight in the fit() method. Returns self : object fit_transform(X, y=None, fit_params) Fit to data, then transform it. Fits transformer to `X` and `y` with optional parameters `fit_params` and returns a transformed version of `X`. Parameters X : array-like of shape (n_samples, n_features) Input samples. y : array-like of shape (n_samples,) or (n_samples, n_outputs), default=None Target values (None for unsupervised transformations). **fit_params : dict Additional fit parameters. Returns X_new : ndarray array of shape (n_samples, n_features_new) Transformed array. get_params(deep=True) Return estimator parameter names for GridSearch support. predict(X) Predict class labels for X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns maj : array-like, shape = [n_samples] Predicted class labels. predict_proba(X) Predict class probabilities for X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns avg : array-like, shape = [n_samples, n_classes] Weighted average probability for each class per sample. score(X, y, sample_weight=None) Return the mean accuracy on the given test data and labels. In multi-label classification, this is the subset accuracy which is a harsh metric since you require for each sample that each label set be correctly predicted. Parameters X : array-like of shape (n_samples, n_features) Test samples. y : array-like of shape (n_samples,) or (n_samples, n_outputs) True labels for X . sample_weight : array-like of shape (n_samples,), default=None Sample weights. Returns score : float Mean accuracy of self.predict(X) wrt. y . set_params( params) Set the parameters of this estimator. The method works on simple estimators as well as on nested objects (such as :class:`~sklearn.pipeline.Pipeline`). The latter have parameters of the form ``<component>__<parameter>`` so that it's possible to update each component of a nested object. Parameters **params : dict Estimator parameters. Returns self : estimator instance Estimator instance. transform(X) Return class labels or probabilities for X for each estimator. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns If voting='soft'`` : array-like = [n_classifiers, n_samples, n_classes] Class probabilties calculated by each classifier. If voting='hard'`` : array-like = [n_classifiers, n_samples] Class labels predicted by each classifier. ython","title":"EnsembleVoteClassifier: A majority voting classifier"},{"location":"user_guide/classifier/EnsembleVoteClassifier/#ensemblevoteclassifier-a-majority-voting-classifier","text":"Implementation of a majority voting EnsembleVoteClassifier for classification. from mlxtend.classifier import EnsembleVoteClassifier","title":"EnsembleVoteClassifier: A majority voting classifier"},{"location":"user_guide/classifier/EnsembleVoteClassifier/#overview","text":"The EnsembleVoteClassifier is a meta-classifier for combining similar or conceptually different machine learning classifiers for classification via majority or plurality voting. (For simplicity, we will refer to both majority and plurality voting as majority voting.) The EnsembleVoteClassifier implements \"hard\" and \"soft\" voting. In hard voting, we predict the final class label as the class label that has been predicted most frequently by the classification models. In soft voting, we predict the class labels by averaging the class-probabilities (only recommended if the classifiers are well-calibrated). Note If you are interested in using the EnsembleVoteClassifier , please note that it is now also available through scikit learn (>0.17) as VotingClassifier .","title":"Overview"},{"location":"user_guide/classifier/EnsembleVoteClassifier/#majority-voting-hard-voting","text":"Hard voting is the simplest case of majority voting. Here, we predict the class label \\hat{y} via majority (plurality) voting of each classifier C_j : \\hat{y}=mode\\{C_1(\\mathbf{x}), C_2(\\mathbf{x}), ..., C_m(\\mathbf{x})\\} Assuming that we combine three classifiers that classify a training sample as follows: classifier 1 -> class 0 classifier 2 -> class 0 classifier 3 -> class 1 \\hat{y}=mode\\{0, 0, 1\\} = 0 Via majority vote, we would classify the sample as \"class 0.\"","title":"Majority Voting / Hard Voting"},{"location":"user_guide/classifier/EnsembleVoteClassifier/#weighted-majority-vote","text":"In addition to the simple majority vote (hard voting) as described in the previous section, we can compute a weighted majority vote by associating a weight w_j with classifier C_j : \\hat{y} = \\arg \\max_i \\sum^{m}_{j=1} w_j \\chi_A \\big(C_j(\\mathbf{x})=i\\big), where \\chi_A is the characteristic function [C_j(\\mathbf{x}) = i \\; \\in A] , and A is the set of unique class labels. Continuing with the example from the previous section classifier 1 -> class 0 classifier 2 -> class 0 classifier 3 -> class 1 assigning the weights {0.2, 0.2, 0.6} would yield a prediction \\hat{y} = 1 : \\arg \\max_i [0.2 \\times i_0 + 0.2 \\times i_0 + 0.6 \\times i_1] = 1","title":"Weighted Majority Vote"},{"location":"user_guide/classifier/EnsembleVoteClassifier/#soft-voting","text":"In soft voting, we predict the class labels based on the predicted probabilities p for classifier -- this approach is only recommended if the classifiers are well-calibrated. \\hat{y} = \\arg \\max_i \\sum^{m}_{j=1} w_j p_{ij}, where w_j is the weight that can be assigned to the j th classifier. Assuming the example in the previous section was a binary classification task with class labels i \\in \\{0, 1\\} , our ensemble could make the following prediction: C_1(\\mathbf{x}) \\rightarrow [0.9, 0.1] C_2(\\mathbf{x}) \\rightarrow [0.8, 0.2] C_3(\\mathbf{x}) \\rightarrow [0.4, 0.6] Using uniform weights, we compute the average probabilities: p(i_0 \\mid \\mathbf{x}) = \\frac{0.9 + 0.8 + 0.4}{3} = 0.7 \\\\\\\\ p(i_1 \\mid \\mathbf{x}) = \\frac{0.1 + 0.2 + 0.6}{3} = 0.3 \\hat{y} = \\arg \\max_i \\big[p(i_0 \\mid \\mathbf{x}), p(i_1 \\mid \\mathbf{x}) \\big] = 0 However, assigning the weights {0.1, 0.1, 0.8} would yield a prediction \\hat{y} = 1 : p(i_0 \\mid \\mathbf{x}) = {0.1 \\times 0.9 + 0.1 \\times 0.8 + 0.8 \\times 0.4} = 0.49 \\\\\\\\ p(i_1 \\mid \\mathbf{x}) = {0.1 \\times 0.1 + 0.2 \\times 0.1 + 0.8 \\times 0.6} = 0.51 \\hat{y} = \\arg \\max_i \\big[p(i_0 \\mid \\mathbf{x}), p(i_1 \\mid \\mathbf{x}) \\big] = 1","title":"Soft Voting"},{"location":"user_guide/classifier/EnsembleVoteClassifier/#references","text":"[1] S. Raschka. Python Machine Learning . Packt Publishing Ltd., 2015.","title":"References"},{"location":"user_guide/classifier/EnsembleVoteClassifier/#example-1-classifying-iris-flowers-using-different-classification-models","text":"from sklearn import datasets iris = datasets.load_iris() X, y = iris.data[:, 1:3], iris.target from sklearn import model_selection from sklearn.linear_model import LogisticRegression from sklearn.naive_bayes import GaussianNB from sklearn.ensemble import RandomForestClassifier import numpy as np clf1 = LogisticRegression(random_state=1) clf2 = RandomForestClassifier(random_state=1) clf3 = GaussianNB() print('5-fold cross validation:\\n') labels = ['Logistic Regression', 'Random Forest', 'Naive Bayes'] for clf, label in zip([clf1, clf2, clf3], labels): scores = model_selection.cross_val_score(clf, X, y, cv=5, scoring='accuracy') print(\"Accuracy: %0.2f (+/- %0.2f) [%s]\" % (scores.mean(), scores.std(), label)) 5-fold cross validation: Accuracy: 0.95 (+/- 0.04) [Logistic Regression] Accuracy: 0.94 (+/- 0.04) [Random Forest] Accuracy: 0.91 (+/- 0.04) [Naive Bayes] from mlxtend.classifier import EnsembleVoteClassifier eclf = EnsembleVoteClassifier(clfs=[clf1, clf2, clf3], weights=[1,1,1]) labels = ['Logistic Regression', 'Random Forest', 'Naive Bayes', 'Ensemble'] for clf, label in zip([clf1, clf2, clf3, eclf], labels): scores = model_selection.cross_val_score(clf, X, y, cv=5, scoring='accuracy') print(\"Accuracy: %0.2f (+/- %0.2f) [%s]\" % (scores.mean(), scores.std(), label)) Accuracy: 0.95 (+/- 0.04) [Logistic Regression] Accuracy: 0.94 (+/- 0.04) [Random Forest] Accuracy: 0.91 (+/- 0.04) [Naive Bayes] Accuracy: 0.95 (+/- 0.04) [Ensemble]","title":"Example 1 -  Classifying Iris Flowers Using Different Classification Models"},{"location":"user_guide/classifier/EnsembleVoteClassifier/#plotting-decision-regions","text":"import matplotlib.pyplot as plt from mlxtend.plotting import plot_decision_regions import matplotlib.gridspec as gridspec import itertools gs = gridspec.GridSpec(2, 2) fig = plt.figure(figsize=(10,8)) labels = ['Logistic Regression', 'Random Forest', 'Naive Bayes', 'Ensemble'] for clf, lab, grd in zip([clf1, clf2, clf3, eclf], labels, itertools.product([0, 1], repeat=2)): clf.fit(X, y) ax = plt.subplot(gs[grd[0], grd[1]]) fig = plot_decision_regions(X=X, y=y, clf=clf) plt.title(lab)","title":"Plotting Decision Regions"},{"location":"user_guide/classifier/EnsembleVoteClassifier/#example-2-grid-search","text":"from sklearn import datasets iris = datasets.load_iris() X, y = iris.data[:, 1:3], iris.target %%capture --no-display from sklearn.model_selection import GridSearchCV from sklearn.linear_model import LogisticRegression from sklearn.naive_bayes import GaussianNB from sklearn.ensemble import RandomForestClassifier from mlxtend.classifier import EnsembleVoteClassifier clf1 = LogisticRegression(random_state=1) clf2 = RandomForestClassifier(random_state=1) clf3 = GaussianNB() eclf = EnsembleVoteClassifier(clfs=[clf1, clf2, clf3], voting='soft') params = {'logisticregression__C': [1.0, 100.0], 'randomforestclassifier__n_estimators': [20, 200],} grid = GridSearchCV(estimator=eclf, param_grid=params, cv=5) grid.fit(iris.data, iris.target) cv_keys = ('mean_test_score', 'std_test_score', 'params') for r, _ in enumerate(grid.cv_results_['mean_test_score']): print(\"%0.3f +/- %0.2f %r\" % (grid.cv_results_[cv_keys[0]][r], grid.cv_results_[cv_keys[1]][r] / 2.0, grid.cv_results_[cv_keys[2]][r])) Note : If the EnsembleClassifier is initialized with multiple similar estimator objects, the estimator names are modified with consecutive integer indices, for example: %%capture --no-display clf1 = LogisticRegression(random_state=1) clf2 = RandomForestClassifier(random_state=1) eclf = EnsembleVoteClassifier(clfs=[clf1, clf1, clf2], voting='soft') params = {'logisticregression-1__C': [1.0, 100.0], 'logisticregression-2__C': [1.0, 100.0], 'randomforestclassifier__n_estimators': [20, 200],} grid = GridSearchCV(estimator=eclf, param_grid=params, cv=5) grid = grid.fit(iris.data, iris.target) Note The EnsembleVoteClass also enables grid search over the clfs argument. However, due to the current implementation of GridSearchCV in scikit-learn, it is not possible to search over both, differenct classifiers and classifier parameters at the same time. For instance, while the following parameter dictionary works params = {'randomforestclassifier__n_estimators': [1, 100], 'clfs': [(clf1, clf1, clf1), (clf2, clf3)]} it will use the instance settings of clf1 , clf2 , and clf3 and not overwrite it with the 'n_estimators' settings from 'randomforestclassifier__n_estimators': [1, 100] .","title":"Example 2 - Grid Search"},{"location":"user_guide/classifier/EnsembleVoteClassifier/#example-3-majority-voting-with-classifiers-trained-on-different-feature-subsets","text":"Feature selection algorithms implemented in scikit-learn as well as the SequentialFeatureSelector implement a transform method that passes the reduced feature subset to the next item in a Pipeline . For example, the method def transform(self, X): return X[:, self.k_feature_idx_] returns the best feature columns, k_feature_idx_ , given a dataset X. Thus, we simply need to construct a Pipeline consisting of the feature selector and the classifier in order to select different feature subsets for different algorithms. During fitting , the optimal feature subsets are automatically determined via the GridSearchCV object, and by calling predict , the fitted feature selector in the pipeline only passes these columns along, which resulted in the best performance for the respective classifier. %%capture --no-display from sklearn import datasets iris = datasets.load_iris() X, y = iris.data[:, :], iris.target from sklearn.model_selection import GridSearchCV from sklearn.linear_model import LogisticRegression from sklearn.naive_bayes import GaussianNB from sklearn.ensemble import RandomForestClassifier from mlxtend.classifier import EnsembleVoteClassifier from sklearn.pipeline import Pipeline from mlxtend.feature_selection import SequentialFeatureSelector clf1 = LogisticRegression(random_state=1) clf2 = RandomForestClassifier(random_state=1) clf3 = GaussianNB() # Creating a feature-selection-classifier pipeline sfs1 = SequentialFeatureSelector(clf1, k_features=4, forward=True, floating=False, scoring='accuracy', verbose=0, cv=0) clf1_pipe = Pipeline([('sfs', sfs1), ('logreg', clf1)]) eclf = EnsembleVoteClassifier(clfs=[clf1_pipe, clf2, clf3], voting='soft') params = {'pipeline__sfs__k_features': [1, 2, 3], 'pipeline__logreg__C': [1.0, 100.0], 'randomforestclassifier__n_estimators': [20, 200]} grid = GridSearchCV(estimator=eclf, param_grid=params, cv=5) grid.fit(iris.data, iris.target) cv_keys = ('mean_test_score', 'std_test_score', 'params') for r, _ in enumerate(grid.cv_results_['mean_test_score']): print(\"%0.3f +/- %0.2f %r\" % (grid.cv_results_[cv_keys[0]][r], grid.cv_results_[cv_keys[1]][r] / 2.0, grid.cv_results_[cv_keys[2]][r])) The best parameters determined via GridSearch are: grid.best_params_ {'pipeline__logreg__C': 1.0, 'pipeline__sfs__k_features': 2, 'randomforestclassifier__n_estimators': 200} Now, we assign these parameters to the ensemble voting classifier, fit the models on the complete training set, and perform a prediction on 3 samples from the Iris dataset. eclf = eclf.set_params(**grid.best_params_) eclf.fit(X, y).predict(X[[1, 51, 149]]) array([0, 1, 2])","title":"Example 3 - Majority voting with classifiers trained on different feature subsets"},{"location":"user_guide/classifier/EnsembleVoteClassifier/#manual-approach","text":"Alternatively, we can select different columns \"manually\" using the ColumnSelector object. In this example, we select only the first (sepal length) and third (petal length) column for the logistic regression classifier ( clf1 ). from mlxtend.feature_selection import ColumnSelector col_sel = ColumnSelector(cols=[0, 2]) clf1_pipe = Pipeline([('sel', col_sel), ('logreg', clf1)]) eclf = EnsembleVoteClassifier(clfs=[clf1_pipe, clf2, clf3], voting='soft') eclf.fit(X, y).predict(X[[1, 51, 149]]) array([0, 1, 2]) Furthermore, we can fit the SequentialFeatureSelector separately, outside the grid search hyperparameter optimization pipeline. Here, we determine the best features first, and then we construct a pipeline using these \"fixed,\" best features as seed for the ColumnSelector : sfs1 = SequentialFeatureSelector(clf1, k_features=2, forward=True, floating=False, scoring='accuracy', verbose=1, cv=0) sfs1.fit(X, y) print('Best features', sfs1.k_feature_idx_) col_sel = ColumnSelector(cols=sfs1.k_feature_idx_) clf1_pipe = Pipeline([('sel', col_sel), ('logreg', clf1)]) [Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers. [Parallel(n_jobs=1)]: Done 4 out of 4 | elapsed: 0.0s finished Features: 1/2[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers. [Parallel(n_jobs=1)]: Done 3 out of 3 | elapsed: 0.0s finished Features: 2/2 Best features (2, 3) eclf = EnsembleVoteClassifier(clfs=[clf1_pipe, clf2, clf3], voting='soft') eclf.fit(X, y).predict(X[[1, 51, 149]]) array([0, 1, 2])","title":"Manual Approach"},{"location":"user_guide/classifier/EnsembleVoteClassifier/#example-5-using-pre-fitted-classifiers","text":"from sklearn import datasets iris = datasets.load_iris() X, y = iris.data[:, 1:3], iris.target Assume that we previously fitted our classifiers: from sklearn import model_selection from sklearn.linear_model import LogisticRegression from sklearn.naive_bayes import GaussianNB from sklearn.ensemble import RandomForestClassifier import numpy as np clf1 = LogisticRegression(random_state=1) clf2 = RandomForestClassifier(random_state=1) clf3 = GaussianNB() for clf in (clf1, clf2, clf3): clf.fit(X, y) By setting fit_base_estimators=False , it will enforce use_clones to be False and the EnsembleVoteClassifier will not re-fit these classifers to save computational time: from mlxtend.classifier import EnsembleVoteClassifier import copy eclf = EnsembleVoteClassifier(clfs=[clf1, clf2, clf3], weights=[1,1,1], fit_base_estimators=False) labels = ['Logistic Regression', 'Random Forest', 'Naive Bayes', 'Ensemble'] eclf.fit(X, y) print('accuracy:', np.mean(y == eclf.predict(X))) accuracy: 0.96 /Users/sebastian/miniforge3/lib/python3.9/site-packages/mlxtend/classifier/ensemble_vote.py:166: UserWarning: fit_base_estimators=False enforces use_clones to be `False` warnings.warn(\"fit_base_estimators=False \" However, please note that fit_base_estimators=False is incompatible to any form of cross-validation that is done in e.g., model_selection.cross_val_score or model_selection.GridSearchCV , etc., since it would require the classifiers to be refit to the training folds. Thus, only use fit_base_estimators=False if you want to make a prediction directly without cross-validation.","title":"Example 5 - Using Pre-fitted Classifiers"},{"location":"user_guide/classifier/EnsembleVoteClassifier/#example-6-ensembles-of-classifiers-that-operate-on-different-feature-subsets","text":"If desired, the different classifiers can be fit to different subsets of features in the training dataset. The following example illustrates how this can be done on a technical level using scikit-learn pipelines and the ColumnSelector : from sklearn.datasets import load_iris from mlxtend.classifier import EnsembleVoteClassifier from mlxtend.feature_selection import ColumnSelector from sklearn.pipeline import make_pipeline from sklearn.linear_model import LogisticRegression iris = load_iris() X = iris.data y = iris.target pipe1 = make_pipeline(ColumnSelector(cols=(0, 2)), LogisticRegression()) pipe2 = make_pipeline(ColumnSelector(cols=(1, 2, 3)), LogisticRegression()) eclf = EnsembleVoteClassifier(clfs=[pipe1, pipe2]) eclf.fit(X, y) EnsembleVoteClassifier(clfs=[Pipeline(steps=[('columnselector', ColumnSelector(cols=(0, 2))), ('logisticregression', LogisticRegression())]), Pipeline(steps=[('columnselector', ColumnSelector(cols=(1, 2, 3))), ('logisticregression', LogisticRegression())])])","title":"Example 6 - Ensembles of Classifiers that Operate on Different Feature Subsets"},{"location":"user_guide/classifier/EnsembleVoteClassifier/#example-7-a-note-about-scikit-learn-svms-and-soft-voting","text":"This section provides some additional technical insights in how probabilities are used when voting='soft' . Note that scikit-learn estimates the probabilities for SVMs (more info here: https://scikit-learn.org/stable/modules/svm.html#scores-probabilities) in a way that these may not be consistent with the class labels that the SVM predicts. This is an extreme example, but let's say we have a dataset with 3 class labels, 0, 1, and 2. For a given training example, the SVM classifier may predict class 2. However, the class-membership probabilities may look as follows: class 0: 99% class 1: 0.5% class 2: 0.5% A practical example of this scenario is shown below: import numpy as np from mlxtend.classifier import EnsembleVoteClassifier from sklearn.svm import SVC from sklearn.datasets import load_iris iris = load_iris() X, y = iris.data, iris.target clf2 = SVC(probability=True, random_state=4) clf2.fit(X, y) eclf = EnsembleVoteClassifier(clfs=[clf2], voting='soft', fit_base_estimators=False) eclf.fit(X, y) for svm_class, e_class, svm_prob, e_prob, in zip(clf2.predict(X), eclf.predict(X), clf2.predict_proba(X), eclf.predict_proba(X)): if svm_class != e_class: print('============') print('Probas from SVM :', svm_prob) print('Class from SVM :', svm_class) print('Probas from SVM in Ensemble:', e_prob) print('Class from SVM in Ensemble :', e_class) print('============') ============ Probas from SVM : [0.00921708 0.49415165 0.49663127] Class from SVM : 1 Probas from SVM in Ensemble: [0.00921708 0.49415165 0.49663127] Class from SVM in Ensemble : 2 ============ /Users/sebastian/miniforge3/lib/python3.9/site-packages/mlxtend/classifier/ensemble_vote.py:166: UserWarning: fit_base_estimators=False enforces use_clones to be `False` warnings.warn(\"fit_base_estimators=False \"","title":"Example 7 - A Note about Scikit-Learn SVMs and Soft Voting"},{"location":"user_guide/classifier/EnsembleVoteClassifier/#example-7-a-note-about-scikit-learn-svms-and-soft-voting_1","text":"Based on the probabilities, we would expect the SVM to predict class 2, because it has the highest probability. Since the EnsembleVoteClassifier uses the argmax function internally if voting='soft' , it would indeed predict class 2 in this case even if the ensemble consists of only one SVM model. Note that in practice, this minor technical detail does not need to concern you, but it is useful to keep it in mind in case you are wondering about results from a 1-model SVM ensemble compared to that SVM alone -- this is not a bug.","title":"Example 7 - A Note about Scikit-Learn SVMs and Soft Voting"},{"location":"user_guide/classifier/EnsembleVoteClassifier/#example-8-optimizing-ensemble-weights-with-nelder-mead","text":"In this section, we will see how we can use a heuristic search method like Nelder-Mead for optimizing the ensemble weights. Suppose we have the following example scenario where we fit 3 individual classifiers on different subsets of the training dataset: from mlxtend.classifier import EnsembleVoteClassifier from mlxtend.data import mnist_data from sklearn.linear_model import LogisticRegression from sklearn.model_selection import train_test_split from sklearn.naive_bayes import GaussianNB from sklearn.tree import DecisionTreeClassifier X, y = mnist_data() X_train, X_val, y_train, y_val = train_test_split( X, y, test_size=0.5, shuffle=True, random_state=1 ) clf1 = GaussianNB() clf2 = LogisticRegression(random_state=123, solver='newton-cg') clf3 = DecisionTreeClassifier(random_state=123, max_depth=2) clf1.fit(X_train[500:1000], y_train[500:1000]) clf2.fit(X_train[750:1250], y_train[750:1250]) clf3.fit(X_train[1250:2000], y_train[1250:2000]); Then, we construct the an ensemble classifier from these 3 classifiers where each classifier contributes equally with weight 1: eclf = EnsembleVoteClassifier( clfs=(clf1, clf2, clf3), voting=\"soft\", # the same would also work with \"hard\" voting weights=(1, 1, 1), use_clones=False, fit_base_estimators=False, ) eclf.fit(X_train, y_train) eclf.score(X_val, y_val) /Users/sebastian/miniforge3/lib/python3.9/site-packages/mlxtend/classifier/ensemble_vote.py:166: UserWarning: fit_base_estimators=False enforces use_clones to be `False` warnings.warn(\"fit_base_estimators=False \" 0.8012 We see that we reach 80% accuracy on the validation set. Can we do better? Maybe they indvidual classifiers shouldn't be contributing equally. Perhaps, we can use an optimization algorothm from scipy.optimize to find a better relative weighting of these individual classifiers. Let's set up an objective function that we want to minimize via SciPy's minimize : from scipy.optimize import minimize def function_to_minimize(weights, fitted_clfs): w1, w2 = weights # these are the new weights! newclf = EnsembleVoteClassifier( voting=\"soft\", use_clones=False, fit_base_estimators=False, clfs=fitted_clfs, weights=(w1, w2, 1.), # use the new weights ) newclf.fit(X_train, y_train) score = newclf.score(X_val, y_val) # change accuracy to error so that smaller is better score_to_minimize = 1 - score return score_to_minimize Note a few things: We only optimize 2 out of the 3 classifier weigths. That's because the weighting is relative to each other, and it would be overkill (and too many degrees of freedom) to also optimize weight 3. We set use_clones=False & fit_base_estimators=False as before, this is to make sure that we use the prefit classifiers in the ensemble classifier. Instead of optimizing the accuracy, we optimize the the classification error, score_to_minimize = 1 - score . That's because we use the minimize function where lower means better. Next, let's choose some initial weight values and run the optimization. Via the bounds we specify the range (lower and upper value) for each weight so that the search doesn't go crazy: %%capture --no-display init_weights = [1., 1.] results = minimize( function_to_minimize, init_weights, args=((clf1, clf2, clf3),), bounds=((0, 5), (0, 5)), method=\"nelder-mead\", ) Let's look at the results! print(results) final_simplex: (array([[0.575 , 1.40625 ], [0.57500153, 1.40622215], [0.57508965, 1.40617647]]), array([0.1324, 0.1324, 0.1324])) fun: 0.13239999999999996 message: 'Optimization terminated successfully.' nfev: 60 nit: 21 status: 0 success: True x: array([0.575 , 1.40625]) It looks like the search was successful and returned the following weights: solution = results[\"x\"] print(solution) [0.575 1.40625] Let's use these new weights in our ensemble classifier: eclf = EnsembleVoteClassifier( clfs=(clf1, clf2, clf3), voting=\"soft\", weights=(solution[0], solution[1], 1), use_clones=False, fit_base_estimators=False, ) eclf.fit(X_train, y_train) eclf.score(X_val, y_val) /Users/sebastian/miniforge3/lib/python3.9/site-packages/mlxtend/classifier/ensemble_vote.py:166: UserWarning: fit_base_estimators=False enforces use_clones to be `False` warnings.warn(\"fit_base_estimators=False \" 0.8676 As we can see, the results on the validation set (0.8676) improved compared to the original ones (0.8012). Yay!","title":"Example 8 - Optimizing Ensemble Weights with Nelder-Mead"},{"location":"user_guide/classifier/EnsembleVoteClassifier/#api","text":"EnsembleVoteClassifier(clfs, voting='hard', weights=None, verbose=0, use_clones=True, fit_base_estimators=True) Soft Voting/Majority Rule classifier for scikit-learn estimators. Parameters clfs : array-like, shape = [n_classifiers] A list of classifiers. Invoking the fit method on the VotingClassifier will fit clones of those original classifiers be stored in the class attribute if use_clones=True (default) and fit_base_estimators=True (default). voting : str, {'hard', 'soft'} (default='hard') If 'hard', uses predicted class labels for majority rule voting. Else if 'soft', predicts the class label based on the argmax of the sums of the predicted probalities, which is recommended for an ensemble of well-calibrated classifiers. weights : array-like, shape = [n_classifiers], optional (default= None ) Sequence of weights ( float or int ) to weight the occurances of predicted class labels ( hard voting) or class probabilities before averaging ( soft voting). Uses uniform weights if None . verbose : int, optional (default=0) Controls the verbosity of the building process. - verbose=0 (default): Prints nothing - verbose=1 : Prints the number & name of the clf being fitted - verbose=2 : Prints info about the parameters of the clf being fitted - verbose>2 : Changes verbose param of the underlying clf to self.verbose - 2 use_clones : bool (default: True) Clones the classifiers for stacking classification if True (default) or else uses the original ones, which will be refitted on the dataset upon calling the fit method. Hence, if use_clones=True, the original input classifiers will remain unmodified upon using the StackingClassifier's fit method. Setting use_clones=False is recommended if you are working with estimators that are supporting the scikit-learn fit/predict API interface but are not compatible to scikit-learn's clone function. fit_base_estimators : bool (default: True) Refits classifiers in clfs if True; uses references to the clfs , otherwise (assumes that the classifiers were already fit). Note: fit_base_estimators=False will enforce use_clones to be False, and is incompatible to most scikit-learn wrappers! For instance, if any form of cross-validation is performed this would require the re-fitting classifiers to training folds, which would raise a NotFitterError if fit_base_estimators=False. (New in mlxtend v0.6.) Attributes classes_ : array-like, shape = [n_predictions] clf : array-like, shape = [n_predictions] The input classifiers; may be overwritten if use_clones=False clf_ : array-like, shape = [n_predictions] Fitted input classifiers; clones if use_clones=True Examples ``` >>> import numpy as np >>> from sklearn.linear_model import LogisticRegression >>> from sklearn.naive_bayes import GaussianNB >>> from sklearn.ensemble import RandomForestClassifier >>> from mlxtend.sklearn import EnsembleVoteClassifier >>> clf1 = LogisticRegression(random_seed=1) >>> clf2 = RandomForestClassifier(random_seed=1) >>> clf3 = GaussianNB() >>> X = np.array([[-1, -1], [-2, -1], [-3, -2], [1, 1], [2, 1], [3, 2]]) >>> y = np.array([1, 1, 1, 2, 2, 2]) >>> eclf1 = EnsembleVoteClassifier(clfs=[clf1, clf2, clf3], ... voting='hard', verbose=1) >>> eclf1 = eclf1.fit(X, y) >>> print(eclf1.predict(X)) [1 1 1 2 2 2] >>> eclf2 = EnsembleVoteClassifier(clfs=[clf1, clf2, clf3], voting='soft') >>> eclf2 = eclf2.fit(X, y) >>> print(eclf2.predict(X)) [1 1 1 2 2 2] >>> eclf3 = EnsembleVoteClassifier(clfs=[clf1, clf2, clf3], ... voting='soft', weights=[2,1,1]) >>> eclf3 = eclf3.fit(X, y) >>> print(eclf3.predict(X)) [1 1 1 2 2 2] >>> For more usage examples, please see https://rasbt.github.io/mlxtend/user_guide/classifier/EnsembleVoteClassifier/ ```","title":"API"},{"location":"user_guide/classifier/EnsembleVoteClassifier/#methods","text":"fit(X, y, sample_weight=None) Learn weight coefficients from training data for each classifier. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values. sample_weight : array-like, shape = [n_samples], optional Sample weights passed as sample_weights to each regressor in the regressors list as well as the meta_regressor. Raises error if some regressor does not support sample_weight in the fit() method. Returns self : object fit_transform(X, y=None, fit_params) Fit to data, then transform it. Fits transformer to `X` and `y` with optional parameters `fit_params` and returns a transformed version of `X`. Parameters X : array-like of shape (n_samples, n_features) Input samples. y : array-like of shape (n_samples,) or (n_samples, n_outputs), default=None Target values (None for unsupervised transformations). **fit_params : dict Additional fit parameters. Returns X_new : ndarray array of shape (n_samples, n_features_new) Transformed array. get_params(deep=True) Return estimator parameter names for GridSearch support. predict(X) Predict class labels for X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns maj : array-like, shape = [n_samples] Predicted class labels. predict_proba(X) Predict class probabilities for X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns avg : array-like, shape = [n_samples, n_classes] Weighted average probability for each class per sample. score(X, y, sample_weight=None) Return the mean accuracy on the given test data and labels. In multi-label classification, this is the subset accuracy which is a harsh metric since you require for each sample that each label set be correctly predicted. Parameters X : array-like of shape (n_samples, n_features) Test samples. y : array-like of shape (n_samples,) or (n_samples, n_outputs) True labels for X . sample_weight : array-like of shape (n_samples,), default=None Sample weights. Returns score : float Mean accuracy of self.predict(X) wrt. y . set_params( params) Set the parameters of this estimator. The method works on simple estimators as well as on nested objects (such as :class:`~sklearn.pipeline.Pipeline`). The latter have parameters of the form ``<component>__<parameter>`` so that it's possible to update each component of a nested object. Parameters **params : dict Estimator parameters. Returns self : estimator instance Estimator instance. transform(X) Return class labels or probabilities for X for each estimator. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns If voting='soft'`` : array-like = [n_classifiers, n_samples, n_classes] Class probabilties calculated by each classifier. If voting='hard'`` : array-like = [n_classifiers, n_samples] Class labels predicted by each classifier. ython","title":"Methods"},{"location":"user_guide/classifier/LogisticRegression/","text":"LogisticRegression: A binary classifier A logistic regression class for binary classification tasks. from mlxtend.classifier import LogisticRegression Overview Related to the Perceptron and 'Adaline' , a Logistic Regression model is a linear model for binary classification. However, instead of minimizing a linear cost function such as the sum of squared errors (SSE) in Adaline, we minimize a sigmoid function, i.e., the logistic function: \\phi(z) = \\frac{1}{1 + e^{-z}}, where z is defined as the net input z = w_0x_0 + w_1x_1 + ... + w_mx_m = \\sum_{j=0}^{m} w_j x_j= \\mathbf{w}^T\\mathbf{x}. The net input is in turn based on the logit function logit(p(y=1 \\mid \\mathbf{x})) = z. Here, p(y=1 \\mid \\mathbf{x}) is the conditional probability that a particular sample belongs to class 1 given its features \\mathbf{x} . The logit function takes inputs in the range [0, 1] and transform them to values over the entire real number range. In contrast, the logistic function takes input values over the entire real number range and transforms them to values in the range [0, 1]. In other words, the logistic function is the inverse of the logit function, and it lets us predict the conditional probability that a certain sample belongs to class 1 (or class 0). After model fitting, the conditional probability p(y=1 \\mid \\mathbf{x}) is converted to a binary class label via a threshold function g(\\cdot) : $$y = g({z}) = \\begin{cases} 1 & \\text{if $\\phi(z) \\ge 0.5$}\\\\ 0 & \\text{otherwise.} \\end{cases} $$ or equivalently: $$y = g({z}) = \\begin{cases} 1 & \\text{if z $\\ge$ 0}\\\\ 0 & \\text{otherwise}. \\end{cases} $$ Objective Function -- Log-Likelihood In order to parameterize a logistic regression model, we maximize the likelihood L(\\cdot) (or minimize the logistic cost function). We write the likelihood as L(\\mathbf{w}) = P(\\mathbf{y} \\mid \\mathbf{x};\\mathbf{w}) = \\prod_{i=1}^{n} P\\big(y^{(i)} \\mid x^{(i)}; \\mathbf{w}\\big) = \\prod^{n}_{i=1}\\bigg(\\phi\\big(z^{(i)}\\big)\\bigg)^{y^{(i)}} \\bigg(1-\\phi\\big(z^{(i)}\\big)\\bigg)^{1-y^{(i)}}, under the assumption that the training samples are independent of each other. In practice, it is easier to maximize the (natural) log of this equation, which is called the log-likelihood function: l(\\mathbf{w}) = \\log L(\\mathbf{w}) = \\sum^{n}_{i=1} y^{(i)} \\log \\bigg(\\phi\\big(z^{(i)}\\big)\\bigg) + \\big( 1 - y^{(i)}\\big) \\log \\big(1-\\phi\\big(z^{(i)}\\big)\\big) One advantage of taking the log is to avoid numeric underflow (and challenges with floating point math) for very small likelihoods. Another advantage is that we can obtain the derivative more easily, using the addition trick to rewrite the product of factors as a summation term, which we can then maximize using optimization algorithms such as gradient ascent. Objective Function -- Logistic Cost Function An alternative to maximizing the log-likelihood, we can define a cost function J(\\cdot) to be minimized; we rewrite the log-likelihood as: J(\\mathbf{w}) = \\sum_{i=1}^{m} - y^{(i)} log \\bigg( \\phi\\big(z^{(i)}\\big) \\bigg) - \\big(1 - y^{(i)}\\big) log\\bigg(1-\\phi\\big(z^{(i)}\\big)\\bigg) $$ J\\big(\\phi(z), y; \\mathbf{w}\\big) = \\begin{cases} -log\\big(\\phi(z) \\big) & \\text{if $y = 1$}\\\\ -log\\big(1- \\phi(z) \\big) & \\text{if $y = 0$} \\end{cases} $$ As we can see in the figure above, we penalize wrong predictions with an increasingly larger cost. Gradient Descent (GD) and Stochastic Gradient Descent (SGD) Optimization Gradient Ascent and the log-likelihood To learn the weight coefficient of a logistic regression model via gradient-based optimization, we compute the partial derivative of the log-likelihood function -- w.r.t. the j th weight -- as follows: \\frac{\\partial}{\\partial w_j} l(\\mathbf{w}) = \\bigg(y \\frac{1}{\\phi(z)} - (1-y) \\frac{1}{1-\\phi{(z)}} \\bigg) \\frac{\\partial}{\\partial w_j}\\phi(z) As an intermediate step, we compute the partial derivative of the sigmoid function, which will come in handy later: \\begin{align} &\\frac{\\partial}{\\partial z} \\phi(z) = \\frac{\\partial}{{\\partial z}} \\frac{1}{1+e^{-z}} \\\\\\\\ &= \\frac{1}{(1 + e^{-z})^{2}} e^{-z}\\\\\\\\ &= \\frac{1}{1+e^{-z}} \\bigg(1 - \\frac{1}{1+e^{-z}} \\bigg)\\\\\\\\ &= \\phi(z)\\big(1-\\phi(z)\\big) \\end{align} Now, we re-substitute \\frac{\\partial}{\\partial z} \\phi(z) = \\phi(z) \\big(1 - \\phi(z)\\big) back into in the log-likelihood partial derivative equation and obtain the equation shown below: \\begin{align} & \\bigg(y \\frac{1}{\\phi{(z)}} - (1 - y) \\frac{1}{1 - \\phi(z)} \\bigg) \\frac{\\partial}{\\partial w_j} \\phi(z) \\\\\\\\ &= \\bigg(y \\frac{1}{\\phi{(z)}} - (1 - y) \\frac{1}{1 - \\phi(z)} \\bigg) \\phi(z) \\big(1 - \\phi(z)\\big) \\frac{\\partial}{\\partial w_j}z\\\\\\\\ &= \\big(y(1-\\phi(z)\\big) - (1 - y) \\phi(z)\\big)x_j\\\\\\\\ &=\\big(y - \\phi(z)\\big)x_j \\end{align} Now, in order to find the weights of the model, we take a step proportional to the positive direction of the gradient to maximize the log-likelihood. Futhermore, we add a coefficient, the learning rate \\eta to the weight update: \\begin{align} & w_j := w_j + \\eta \\frac{\\partial}{\\partial w_j} l(\\mathbf{w})\\\\\\\\ & w_j := w_j + \\eta \\sum^{n}_{i=1} \\big( y^{(i)} - \\phi\\big(z^{(i)}\\big)\\big)x_j^{(i)} \\end{align} Note that the gradient (and weight update) is computed from all samples in the training set in gradient ascent/descent in contrast to stochastic gradient ascent/descent. For more information about the differences between gradient descent and stochastic gradient descent, please see the related article Gradient Descent and Stochastic Gradient Descent . The previous equation shows the weight update for a single weight j . In gradient-based optimization, all weight coefficients are updated simultaneously; the weight update can be written more compactly as \\mathbf{w} := \\mathbf{w} + \\Delta\\mathbf{w}, where \\Delta{\\mathbf{w}} = \\eta \\nabla l(\\mathbf{w}) Gradient Descent and the logistic cost function In the previous section, we derived the gradient of the log-likelihood function, which can be optimized via gradient ascent. Similarly, we can obtain the cost gradient of the logistic cost function J(\\cdot) and minimize it via gradient descent in order to learn the logistic regression model. The update rule for a single weight: \\begin{align} & \\Delta{w_j} = -\\eta \\frac{\\partial J}{\\partial w_j} \\\\ & = - \\eta \\sum_{i=1}^{n}\\big(y^{(i)} - \\phi\\big(z^{(i)}\\big) x^{(i)} \\big) \\end{align} The simultaneous weight update: \\mathbf{w} := \\mathbf{w} + \\Delta\\mathbf{w} where \\Delta{\\mathbf{w}} = - \\eta \\nabla J(\\mathbf{w}). Shuffling Random shuffling is implemented as: for one or more epochs randomly shuffle samples in the training set for training sample i compute gradients and perform weight updates Regularization As a way to tackle overfitting, we can add additional bias to the logistic regression model via a regularization terms. Via the L2 regularization term, we reduce the complexity of the model by penalizing large weight coefficients: L2: \\frac{\\lambda}{2}\\lVert \\mathbf{w} \\lVert_2 = \\frac{\\lambda}{2} \\sum_{j=1}^{m} w_j^2 In order to apply regularization, we just need to add the regularization term to the cost function that we defined for logistic regression to shrink the weights: J(\\mathbf{w}) = \\sum_{i=1}^{m} \\Bigg[ - y^{(i)} log \\bigg( \\phi\\big(z^{(i)}\\big) \\bigg) - \\big(1 - y^{(i)}\\big) log\\bigg(1-\\phi\\big(z^{(i)}\\big)\\bigg) \\Bigg] + \\frac{\\lambda}{2} \\sum_{j=1}^{m} w_j^2 The update rule for a single weight: \\begin{align} & \\Delta{w_j} = -\\eta \\bigg( \\frac{\\partial J}{\\partial w_j} + \\lambda w_j\\bigg)\\\\ & = - \\eta \\sum_{i=1}^{n}\\big(y^{(i)} - \\phi\\big(z^{(i)}\\big) x^{(i)} \\big) - \\eta \\lambda w_j \\end{align} The simultaneous weight update: \\mathbf{w} := \\mathbf{w} + \\Delta\\mathbf{w} where \\Delta{\\mathbf{w}} = - \\eta \\big( \\nabla J(\\mathbf{w}) + \\lambda \\mathbf{w}\\big). For more information on regularization, please see Regularization of Generalized Linear Models . References Bishop, Christopher M. Pattern recognition and machine learning . Springer, 2006. pp. 203-213 Example 1 - Gradient Descent from mlxtend.data import iris_data from mlxtend.plotting import plot_decision_regions from mlxtend.classifier import LogisticRegression import matplotlib.pyplot as plt # Loading Data X, y = iris_data() X = X[:, [0, 3]] # sepal length and petal width X = X[0:100] # class 0 and class 1 y = y[0:100] # class 0 and class 1 # standardize X[:,0] = (X[:,0] - X[:,0].mean()) / X[:,0].std() X[:,1] = (X[:,1] - X[:,1].mean()) / X[:,1].std() lr = LogisticRegression(eta=0.1, l2_lambda=0.0, epochs=100, minibatches=1, # for Gradient Descent random_seed=1, print_progress=3) lr.fit(X, y) plot_decision_regions(X, y, clf=lr) plt.title('Logistic Regression - Gradient Descent') plt.show() plt.plot(range(len(lr.cost_)), lr.cost_) plt.xlabel('Iterations') plt.ylabel('Cost') plt.show() Iteration: 100/100 | Cost 0.32 | Elapsed: 0:00:00 | ETA: 0:00:00 Predicting Class Labels y_pred = lr.predict(X) print('Last 3 Class Labels: %s' % y_pred[-3:]) Last 3 Class Labels: [1 1 1] Predicting Class Probabilities y_pred = lr.predict_proba(X) print('Last 3 Class Labels: %s' % y_pred[-3:]) Last 3 Class Labels: [ 0.99997968 0.99339873 0.99992707] Example 2 - Stochastic Gradient Descent from mlxtend.data import iris_data from mlxtend.plotting import plot_decision_regions from mlxtend.classifier import LogisticRegression import matplotlib.pyplot as plt # Loading Data X, y = iris_data() X = X[:, [0, 3]] # sepal length and petal width X = X[0:100] # class 0 and class 1 y = y[0:100] # class 0 and class 1 # standardize X[:,0] = (X[:,0] - X[:,0].mean()) / X[:,0].std() X[:,1] = (X[:,1] - X[:,1].mean()) / X[:,1].std() lr = LogisticRegression(eta=0.5, epochs=30, l2_lambda=0.0, minibatches=len(y), # for SGD learning random_seed=1, print_progress=3) lr.fit(X, y) plot_decision_regions(X, y, clf=lr) plt.title('Logistic Regression - Stochastic Gradient Descent') plt.show() plt.plot(range(len(lr.cost_)), lr.cost_) plt.xlabel('Iterations') plt.ylabel('Cost') plt.show() Iteration: 30/30 | Cost 0.27 | Elapsed: 0:00:00 | ETA: 0:00:00 Example 3 - Stochastic Gradient Descent w. Minibatches Here, we set minibatches to 5, which will result in Minibatch Learning with a batch size of 20 samples (since 100 Iris samples divided by 5 minibatches equals 20). from mlxtend.data import iris_data from mlxtend.plotting import plot_decision_regions from mlxtend.classifier import LogisticRegression import matplotlib.pyplot as plt # Loading Data X, y = iris_data() X = X[:, [0, 3]] # sepal length and petal width X = X[0:100] # class 0 and class 1 y = y[0:100] # class 0 and class 1 # standardize X[:,0] = (X[:,0] - X[:,0].mean()) / X[:,0].std() X[:,1] = (X[:,1] - X[:,1].mean()) / X[:,1].std() lr = LogisticRegression(eta=0.5, epochs=30, l2_lambda=0.0, minibatches=5, # 100/5 = 20 -> minibatch-s random_seed=1, print_progress=3) lr.fit(X, y) plot_decision_regions(X, y, clf=lr) plt.title('Logistic Regression - Stochastic Gradient Descent') plt.show() plt.plot(range(len(lr.cost_)), lr.cost_) plt.xlabel('Iterations') plt.ylabel('Cost') plt.show() Iteration: 30/30 | Cost 0.25 | Elapsed: 0:00:00 | ETA: 0:00:00 API LogisticRegression(eta=0.01, epochs=50, l2_lambda=0.0, minibatches=1, random_seed=None, print_progress=0) Logistic regression classifier. Note that this implementation of Logistic Regression expects binary class labels in {0, 1}. Parameters eta : float (default: 0.01) Learning rate (between 0.0 and 1.0) epochs : int (default: 50) Passes over the training dataset. Prior to each epoch, the dataset is shuffled if minibatches > 1 to prevent cycles in stochastic gradient descent. l2_lambda : float Regularization parameter for L2 regularization. No regularization if l2_lambda=0.0. minibatches : int (default: 1) The number of minibatches for gradient-based optimization. If 1: Gradient Descent learning If len(y): Stochastic Gradient Descent (SGD) online learning If 1 < minibatches < len(y): SGD Minibatch learning random_seed : int (default: None) Set random state for shuffling and initializing the weights. print_progress : int (default: 0) Prints progress in fitting to stderr. 0: No output 1: Epochs elapsed and cost 2: 1 plus time elapsed 3: 2 plus estimated time until completion Attributes w_ : 2d-array, shape={n_features, 1} Model weights after fitting. b_ : 1d-array, shape={1,} Bias unit after fitting. cost_ : list List of floats with cross_entropy cost (sgd or gd) for every epoch. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/classifier/LogisticRegression/ Methods fit(X, y, init_params=True) Learn model from training data. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values. init_params : bool (default: True) Re-initializes model parameters prior to fitting. Set False to continue training with weights from a previous model fitting. Returns self : object predict(X) Predict targets from X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns target_values : array-like, shape = [n_samples] Predicted target values. predict_proba(X) Predict class probabilities of X from the net input. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns Class 1 probability : float score(X, y) Compute the prediction accuracy Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values (true class labels). Returns acc : float The prediction accuracy as a float between 0.0 and 1.0 (perfect score).","title":"LogisticRegression: A binary classifier"},{"location":"user_guide/classifier/LogisticRegression/#logisticregression-a-binary-classifier","text":"A logistic regression class for binary classification tasks. from mlxtend.classifier import LogisticRegression","title":"LogisticRegression: A binary classifier"},{"location":"user_guide/classifier/LogisticRegression/#overview","text":"Related to the Perceptron and 'Adaline' , a Logistic Regression model is a linear model for binary classification. However, instead of minimizing a linear cost function such as the sum of squared errors (SSE) in Adaline, we minimize a sigmoid function, i.e., the logistic function: \\phi(z) = \\frac{1}{1 + e^{-z}}, where z is defined as the net input z = w_0x_0 + w_1x_1 + ... + w_mx_m = \\sum_{j=0}^{m} w_j x_j= \\mathbf{w}^T\\mathbf{x}. The net input is in turn based on the logit function logit(p(y=1 \\mid \\mathbf{x})) = z. Here, p(y=1 \\mid \\mathbf{x}) is the conditional probability that a particular sample belongs to class 1 given its features \\mathbf{x} . The logit function takes inputs in the range [0, 1] and transform them to values over the entire real number range. In contrast, the logistic function takes input values over the entire real number range and transforms them to values in the range [0, 1]. In other words, the logistic function is the inverse of the logit function, and it lets us predict the conditional probability that a certain sample belongs to class 1 (or class 0). After model fitting, the conditional probability p(y=1 \\mid \\mathbf{x}) is converted to a binary class label via a threshold function g(\\cdot) : $$y = g({z}) = \\begin{cases} 1 & \\text{if $\\phi(z) \\ge 0.5$}\\\\ 0 & \\text{otherwise.} \\end{cases} $$ or equivalently: $$y = g({z}) = \\begin{cases} 1 & \\text{if z $\\ge$ 0}\\\\ 0 & \\text{otherwise}. \\end{cases} $$","title":"Overview"},{"location":"user_guide/classifier/LogisticRegression/#objective-function-log-likelihood","text":"In order to parameterize a logistic regression model, we maximize the likelihood L(\\cdot) (or minimize the logistic cost function). We write the likelihood as L(\\mathbf{w}) = P(\\mathbf{y} \\mid \\mathbf{x};\\mathbf{w}) = \\prod_{i=1}^{n} P\\big(y^{(i)} \\mid x^{(i)}; \\mathbf{w}\\big) = \\prod^{n}_{i=1}\\bigg(\\phi\\big(z^{(i)}\\big)\\bigg)^{y^{(i)}} \\bigg(1-\\phi\\big(z^{(i)}\\big)\\bigg)^{1-y^{(i)}}, under the assumption that the training samples are independent of each other. In practice, it is easier to maximize the (natural) log of this equation, which is called the log-likelihood function: l(\\mathbf{w}) = \\log L(\\mathbf{w}) = \\sum^{n}_{i=1} y^{(i)} \\log \\bigg(\\phi\\big(z^{(i)}\\big)\\bigg) + \\big( 1 - y^{(i)}\\big) \\log \\big(1-\\phi\\big(z^{(i)}\\big)\\big) One advantage of taking the log is to avoid numeric underflow (and challenges with floating point math) for very small likelihoods. Another advantage is that we can obtain the derivative more easily, using the addition trick to rewrite the product of factors as a summation term, which we can then maximize using optimization algorithms such as gradient ascent.","title":"Objective Function -- Log-Likelihood"},{"location":"user_guide/classifier/LogisticRegression/#objective-function-logistic-cost-function","text":"An alternative to maximizing the log-likelihood, we can define a cost function J(\\cdot) to be minimized; we rewrite the log-likelihood as: J(\\mathbf{w}) = \\sum_{i=1}^{m} - y^{(i)} log \\bigg( \\phi\\big(z^{(i)}\\big) \\bigg) - \\big(1 - y^{(i)}\\big) log\\bigg(1-\\phi\\big(z^{(i)}\\big)\\bigg) $$ J\\big(\\phi(z), y; \\mathbf{w}\\big) = \\begin{cases} -log\\big(\\phi(z) \\big) & \\text{if $y = 1$}\\\\ -log\\big(1- \\phi(z) \\big) & \\text{if $y = 0$} \\end{cases} $$ As we can see in the figure above, we penalize wrong predictions with an increasingly larger cost.","title":"Objective Function -- Logistic Cost Function"},{"location":"user_guide/classifier/LogisticRegression/#gradient-descent-gd-and-stochastic-gradient-descent-sgd-optimization","text":"","title":"Gradient Descent (GD)  and Stochastic Gradient Descent (SGD) Optimization"},{"location":"user_guide/classifier/LogisticRegression/#gradient-ascent-and-the-log-likelihood","text":"To learn the weight coefficient of a logistic regression model via gradient-based optimization, we compute the partial derivative of the log-likelihood function -- w.r.t. the j th weight -- as follows: \\frac{\\partial}{\\partial w_j} l(\\mathbf{w}) = \\bigg(y \\frac{1}{\\phi(z)} - (1-y) \\frac{1}{1-\\phi{(z)}} \\bigg) \\frac{\\partial}{\\partial w_j}\\phi(z) As an intermediate step, we compute the partial derivative of the sigmoid function, which will come in handy later: \\begin{align} &\\frac{\\partial}{\\partial z} \\phi(z) = \\frac{\\partial}{{\\partial z}} \\frac{1}{1+e^{-z}} \\\\\\\\ &= \\frac{1}{(1 + e^{-z})^{2}} e^{-z}\\\\\\\\ &= \\frac{1}{1+e^{-z}} \\bigg(1 - \\frac{1}{1+e^{-z}} \\bigg)\\\\\\\\ &= \\phi(z)\\big(1-\\phi(z)\\big) \\end{align} Now, we re-substitute \\frac{\\partial}{\\partial z} \\phi(z) = \\phi(z) \\big(1 - \\phi(z)\\big) back into in the log-likelihood partial derivative equation and obtain the equation shown below: \\begin{align} & \\bigg(y \\frac{1}{\\phi{(z)}} - (1 - y) \\frac{1}{1 - \\phi(z)} \\bigg) \\frac{\\partial}{\\partial w_j} \\phi(z) \\\\\\\\ &= \\bigg(y \\frac{1}{\\phi{(z)}} - (1 - y) \\frac{1}{1 - \\phi(z)} \\bigg) \\phi(z) \\big(1 - \\phi(z)\\big) \\frac{\\partial}{\\partial w_j}z\\\\\\\\ &= \\big(y(1-\\phi(z)\\big) - (1 - y) \\phi(z)\\big)x_j\\\\\\\\ &=\\big(y - \\phi(z)\\big)x_j \\end{align} Now, in order to find the weights of the model, we take a step proportional to the positive direction of the gradient to maximize the log-likelihood. Futhermore, we add a coefficient, the learning rate \\eta to the weight update: \\begin{align} & w_j := w_j + \\eta \\frac{\\partial}{\\partial w_j} l(\\mathbf{w})\\\\\\\\ & w_j := w_j + \\eta \\sum^{n}_{i=1} \\big( y^{(i)} - \\phi\\big(z^{(i)}\\big)\\big)x_j^{(i)} \\end{align} Note that the gradient (and weight update) is computed from all samples in the training set in gradient ascent/descent in contrast to stochastic gradient ascent/descent. For more information about the differences between gradient descent and stochastic gradient descent, please see the related article Gradient Descent and Stochastic Gradient Descent . The previous equation shows the weight update for a single weight j . In gradient-based optimization, all weight coefficients are updated simultaneously; the weight update can be written more compactly as \\mathbf{w} := \\mathbf{w} + \\Delta\\mathbf{w}, where \\Delta{\\mathbf{w}} = \\eta \\nabla l(\\mathbf{w})","title":"Gradient Ascent and the log-likelihood"},{"location":"user_guide/classifier/LogisticRegression/#gradient-descent-and-the-logistic-cost-function","text":"In the previous section, we derived the gradient of the log-likelihood function, which can be optimized via gradient ascent. Similarly, we can obtain the cost gradient of the logistic cost function J(\\cdot) and minimize it via gradient descent in order to learn the logistic regression model. The update rule for a single weight: \\begin{align} & \\Delta{w_j} = -\\eta \\frac{\\partial J}{\\partial w_j} \\\\ & = - \\eta \\sum_{i=1}^{n}\\big(y^{(i)} - \\phi\\big(z^{(i)}\\big) x^{(i)} \\big) \\end{align} The simultaneous weight update: \\mathbf{w} := \\mathbf{w} + \\Delta\\mathbf{w} where \\Delta{\\mathbf{w}} = - \\eta \\nabla J(\\mathbf{w}).","title":"Gradient Descent and the logistic cost function"},{"location":"user_guide/classifier/LogisticRegression/#shuffling","text":"Random shuffling is implemented as: for one or more epochs randomly shuffle samples in the training set for training sample i compute gradients and perform weight updates","title":"Shuffling"},{"location":"user_guide/classifier/LogisticRegression/#regularization","text":"As a way to tackle overfitting, we can add additional bias to the logistic regression model via a regularization terms. Via the L2 regularization term, we reduce the complexity of the model by penalizing large weight coefficients: L2: \\frac{\\lambda}{2}\\lVert \\mathbf{w} \\lVert_2 = \\frac{\\lambda}{2} \\sum_{j=1}^{m} w_j^2 In order to apply regularization, we just need to add the regularization term to the cost function that we defined for logistic regression to shrink the weights: J(\\mathbf{w}) = \\sum_{i=1}^{m} \\Bigg[ - y^{(i)} log \\bigg( \\phi\\big(z^{(i)}\\big) \\bigg) - \\big(1 - y^{(i)}\\big) log\\bigg(1-\\phi\\big(z^{(i)}\\big)\\bigg) \\Bigg] + \\frac{\\lambda}{2} \\sum_{j=1}^{m} w_j^2 The update rule for a single weight: \\begin{align} & \\Delta{w_j} = -\\eta \\bigg( \\frac{\\partial J}{\\partial w_j} + \\lambda w_j\\bigg)\\\\ & = - \\eta \\sum_{i=1}^{n}\\big(y^{(i)} - \\phi\\big(z^{(i)}\\big) x^{(i)} \\big) - \\eta \\lambda w_j \\end{align} The simultaneous weight update: \\mathbf{w} := \\mathbf{w} + \\Delta\\mathbf{w} where \\Delta{\\mathbf{w}} = - \\eta \\big( \\nabla J(\\mathbf{w}) + \\lambda \\mathbf{w}\\big). For more information on regularization, please see Regularization of Generalized Linear Models .","title":"Regularization"},{"location":"user_guide/classifier/LogisticRegression/#references","text":"Bishop, Christopher M. Pattern recognition and machine learning . Springer, 2006. pp. 203-213","title":"References"},{"location":"user_guide/classifier/LogisticRegression/#example-1-gradient-descent","text":"from mlxtend.data import iris_data from mlxtend.plotting import plot_decision_regions from mlxtend.classifier import LogisticRegression import matplotlib.pyplot as plt # Loading Data X, y = iris_data() X = X[:, [0, 3]] # sepal length and petal width X = X[0:100] # class 0 and class 1 y = y[0:100] # class 0 and class 1 # standardize X[:,0] = (X[:,0] - X[:,0].mean()) / X[:,0].std() X[:,1] = (X[:,1] - X[:,1].mean()) / X[:,1].std() lr = LogisticRegression(eta=0.1, l2_lambda=0.0, epochs=100, minibatches=1, # for Gradient Descent random_seed=1, print_progress=3) lr.fit(X, y) plot_decision_regions(X, y, clf=lr) plt.title('Logistic Regression - Gradient Descent') plt.show() plt.plot(range(len(lr.cost_)), lr.cost_) plt.xlabel('Iterations') plt.ylabel('Cost') plt.show() Iteration: 100/100 | Cost 0.32 | Elapsed: 0:00:00 | ETA: 0:00:00","title":"Example 1 - Gradient Descent"},{"location":"user_guide/classifier/LogisticRegression/#predicting-class-labels","text":"y_pred = lr.predict(X) print('Last 3 Class Labels: %s' % y_pred[-3:]) Last 3 Class Labels: [1 1 1]","title":"Predicting Class Labels"},{"location":"user_guide/classifier/LogisticRegression/#predicting-class-probabilities","text":"y_pred = lr.predict_proba(X) print('Last 3 Class Labels: %s' % y_pred[-3:]) Last 3 Class Labels: [ 0.99997968 0.99339873 0.99992707]","title":"Predicting Class Probabilities"},{"location":"user_guide/classifier/LogisticRegression/#example-2-stochastic-gradient-descent","text":"from mlxtend.data import iris_data from mlxtend.plotting import plot_decision_regions from mlxtend.classifier import LogisticRegression import matplotlib.pyplot as plt # Loading Data X, y = iris_data() X = X[:, [0, 3]] # sepal length and petal width X = X[0:100] # class 0 and class 1 y = y[0:100] # class 0 and class 1 # standardize X[:,0] = (X[:,0] - X[:,0].mean()) / X[:,0].std() X[:,1] = (X[:,1] - X[:,1].mean()) / X[:,1].std() lr = LogisticRegression(eta=0.5, epochs=30, l2_lambda=0.0, minibatches=len(y), # for SGD learning random_seed=1, print_progress=3) lr.fit(X, y) plot_decision_regions(X, y, clf=lr) plt.title('Logistic Regression - Stochastic Gradient Descent') plt.show() plt.plot(range(len(lr.cost_)), lr.cost_) plt.xlabel('Iterations') plt.ylabel('Cost') plt.show() Iteration: 30/30 | Cost 0.27 | Elapsed: 0:00:00 | ETA: 0:00:00","title":"Example 2 - Stochastic Gradient Descent"},{"location":"user_guide/classifier/LogisticRegression/#example-3-stochastic-gradient-descent-w-minibatches","text":"Here, we set minibatches to 5, which will result in Minibatch Learning with a batch size of 20 samples (since 100 Iris samples divided by 5 minibatches equals 20). from mlxtend.data import iris_data from mlxtend.plotting import plot_decision_regions from mlxtend.classifier import LogisticRegression import matplotlib.pyplot as plt # Loading Data X, y = iris_data() X = X[:, [0, 3]] # sepal length and petal width X = X[0:100] # class 0 and class 1 y = y[0:100] # class 0 and class 1 # standardize X[:,0] = (X[:,0] - X[:,0].mean()) / X[:,0].std() X[:,1] = (X[:,1] - X[:,1].mean()) / X[:,1].std() lr = LogisticRegression(eta=0.5, epochs=30, l2_lambda=0.0, minibatches=5, # 100/5 = 20 -> minibatch-s random_seed=1, print_progress=3) lr.fit(X, y) plot_decision_regions(X, y, clf=lr) plt.title('Logistic Regression - Stochastic Gradient Descent') plt.show() plt.plot(range(len(lr.cost_)), lr.cost_) plt.xlabel('Iterations') plt.ylabel('Cost') plt.show() Iteration: 30/30 | Cost 0.25 | Elapsed: 0:00:00 | ETA: 0:00:00","title":"Example 3 - Stochastic Gradient Descent w. Minibatches"},{"location":"user_guide/classifier/LogisticRegression/#api","text":"LogisticRegression(eta=0.01, epochs=50, l2_lambda=0.0, minibatches=1, random_seed=None, print_progress=0) Logistic regression classifier. Note that this implementation of Logistic Regression expects binary class labels in {0, 1}. Parameters eta : float (default: 0.01) Learning rate (between 0.0 and 1.0) epochs : int (default: 50) Passes over the training dataset. Prior to each epoch, the dataset is shuffled if minibatches > 1 to prevent cycles in stochastic gradient descent. l2_lambda : float Regularization parameter for L2 regularization. No regularization if l2_lambda=0.0. minibatches : int (default: 1) The number of minibatches for gradient-based optimization. If 1: Gradient Descent learning If len(y): Stochastic Gradient Descent (SGD) online learning If 1 < minibatches < len(y): SGD Minibatch learning random_seed : int (default: None) Set random state for shuffling and initializing the weights. print_progress : int (default: 0) Prints progress in fitting to stderr. 0: No output 1: Epochs elapsed and cost 2: 1 plus time elapsed 3: 2 plus estimated time until completion Attributes w_ : 2d-array, shape={n_features, 1} Model weights after fitting. b_ : 1d-array, shape={1,} Bias unit after fitting. cost_ : list List of floats with cross_entropy cost (sgd or gd) for every epoch. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/classifier/LogisticRegression/","title":"API"},{"location":"user_guide/classifier/LogisticRegression/#methods","text":"fit(X, y, init_params=True) Learn model from training data. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values. init_params : bool (default: True) Re-initializes model parameters prior to fitting. Set False to continue training with weights from a previous model fitting. Returns self : object predict(X) Predict targets from X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns target_values : array-like, shape = [n_samples] Predicted target values. predict_proba(X) Predict class probabilities of X from the net input. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns Class 1 probability : float score(X, y) Compute the prediction accuracy Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values (true class labels). Returns acc : float The prediction accuracy as a float between 0.0 and 1.0 (perfect score).","title":"Methods"},{"location":"user_guide/classifier/MultiLayerPerceptron/","text":"MultilayerPerceptron: A simple multilayer neural network Implementation of a multilayer perceptron, a feedforward artificial neural network. from mlxtend.classifier import MultiLayerPerceptron Overview Although the code is fully working and can be used for common classification tasks, this implementation is not geared towards efficiency but clarity \u2013 the original code was written for demonstration purposes. Basic Architecture The neurons x_0 and a_0 represent the bias units ( x_0=1 , a_0=1 ). The i th superscript denotes the i th layer, and the j th subscripts stands for the index of the respective unit. For example, a_{1}^{(2)} refers to the first activation unit after the bias unit (i.e., 2nd activation unit) in the 2nd layer (here: the hidden layer) \\begin{align} \\mathbf{a^{(2)}} &= \\begin{bmatrix} a_{0}^{(2)} \\\\ a_{1}^{(2)} \\\\ \\vdots \\\\ a_{m}^{(2)} \\end{bmatrix}. \\end{align} Each layer (l) in a multi-layer perceptron, a directed graph, is fully connected to the next layer (l+1) . We write the weight coefficient that connects the k th unit in the l th layer to the j th unit in layer l+1 as w^{(l)}_{j, k} . For example, the weight coefficient that connects the units a_0^{(2)} \\rightarrow a_1^{(3)} would be written as w_{1,0}^{(2)} . Activation In the current implementation, the activations of the hidden layer(s) are computed via the logistic (sigmoid) function \\phi(z) = \\frac{1}{1 + e^{-z}}. (For more details on the logistic function, please see classifier.LogisticRegression ; a general overview of different activation function can be found here .) Furthermore, the MLP uses the softmax function in the output layer, For more details on the logistic function, please see classifier.SoftmaxRegression . References D. R. G. H. R. Williams and G. Hinton. Learning representations by back-propagating errors . Nature, pages 323\u2013533, 1986. C. M. Bishop. Neural networks for pattern recognition . Oxford University Press, 1995. T. Hastie, J. Friedman, and R. Tibshirani. The Elements of Statistical Learning , Volume 2. Springer, 2009. Example 1 - Classifying Iris Flowers Load 2 features from Iris (petal length and petal width) for visualization purposes: from mlxtend.data import iris_data X, y = iris_data() X = X[:, [0, 3]] # standardize training data X_std = (X - X.mean(axis=0)) / X.std(axis=0) Train neural network for 3 output flower classes ('Setosa', 'Versicolor', 'Virginica'), regular gradient decent ( minibatches=1 ), 30 hidden units, and no regularization. Gradient Descent Setting the minibatches to 1 will result in gradient descent training; please see Gradient Descent vs. Stochastic Gradient Descent for details. from mlxtend.classifier import MultiLayerPerceptron as MLP nn1 = MLP(hidden_layers=[50], l2=0.00, l1=0.0, epochs=150, eta=0.05, momentum=0.1, decrease_const=0.0, minibatches=1, random_seed=1, print_progress=3) nn1 = nn1.fit(X_std, y) Iteration: 150/150 | Cost 0.06 | Elapsed: 0:00:00 | ETA: 0:00:00 from mlxtend.plotting import plot_decision_regions import matplotlib.pyplot as plt fig = plot_decision_regions(X=X_std, y=y, clf=nn1, legend=2) plt.title('Multi-layer Perceptron w. 1 hidden layer (logistic sigmoid)') plt.show() import matplotlib.pyplot as plt plt.plot(range(len(nn1.cost_)), nn1.cost_) plt.ylabel('Cost') plt.xlabel('Epochs') plt.show() print('Accuracy: %.2f%%' % (100 * nn1.score(X_std, y))) Accuracy: 96.67% Stochastic Gradient Descent Setting minibatches to n_samples will result in stochastic gradient descent training; please see Gradient Descent vs. Stochastic Gradient Descent for details. nn2 = MLP(hidden_layers=[50], l2=0.00, l1=0.0, epochs=5, eta=0.005, momentum=0.1, decrease_const=0.0, minibatches=len(y), random_seed=1, print_progress=3) nn2.fit(X_std, y) plt.plot(range(len(nn2.cost_)), nn2.cost_) plt.ylabel('Cost') plt.xlabel('Epochs') plt.show() Iteration: 5/5 | Cost 0.11 | Elapsed: 00:00:00 | ETA: 00:00:00 Continue the training for 25 epochs... nn2.epochs = 25 nn2 = nn2.fit(X_std, y) Iteration: 25/25 | Cost 0.07 | Elapsed: 0:00:00 | ETA: 0:00:00 plt.plot(range(len(nn2.cost_)), nn2.cost_) plt.ylabel('Cost') plt.xlabel('Epochs') plt.show() Example 2 - Classifying Handwritten Digits from a 10% MNIST Subset Load a 5000-sample subset of the MNIST dataset (please see data.loadlocal_mnist if you want to download and read in the complete MNIST dataset). from mlxtend.data import mnist_data from mlxtend.preprocessing import shuffle_arrays_unison X, y = mnist_data() X, y = shuffle_arrays_unison((X, y), random_seed=1) X_train, y_train = X[:500], y[:500] X_test, y_test = X[500:], y[500:] Visualize a sample from the MNIST dataset to check if it was loaded correctly: import matplotlib.pyplot as plt def plot_digit(X, y, idx): img = X[idx].reshape(28,28) plt.imshow(img, cmap='Greys', interpolation='nearest') plt.title('true label: %d' % y[idx]) plt.show() plot_digit(X, y, 3500) Standardize pixel values: import numpy as np from mlxtend.preprocessing import standardize X_train_std, params = standardize(X_train, columns=range(X_train.shape[1]), return_params=True) X_test_std = standardize(X_test, columns=range(X_test.shape[1]), params=params) Initialize the neural network to recognize the 10 different digits (0-10) using 300 epochs and mini-batch learning. nn1 = MLP(hidden_layers=[150], l2=0.00, l1=0.0, epochs=100, eta=0.005, momentum=0.0, decrease_const=0.0, minibatches=100, random_seed=1, print_progress=3) Learn the features while printing the progress to get an idea about how long it may take. import matplotlib.pyplot as plt nn1.fit(X_train_std, y_train) plt.plot(range(len(nn1.cost_)), nn1.cost_) plt.ylabel('Cost') plt.xlabel('Epochs') plt.show() Iteration: 100/100 | Cost 0.01 | Elapsed: 0:00:17 | ETA: 0:00:00 print('Train Accuracy: %.2f%%' % (100 * nn1.score(X_train_std, y_train))) print('Test Accuracy: %.2f%%' % (100 * nn1.score(X_test_std, y_test))) Train Accuracy: 100.00% Test Accuracy: 84.62% Please note that this neural network has been trained on only 10% of the MNIST data for technical demonstration purposes, hence, the lousy predictive performance. API MultiLayerPerceptron(eta=0.5, epochs=50, hidden_layers=[50], n_classes=None, momentum=0.0, l1=0.0, l2=0.0, dropout=1.0, decrease_const=0.0, minibatches=1, random_seed=None, print_progress=0) Multi-layer perceptron classifier with logistic sigmoid activations Parameters eta : float (default: 0.5) Learning rate (between 0.0 and 1.0) epochs : int (default: 50) Passes over the training dataset. Prior to each epoch, the dataset is shuffled if minibatches > 1 to prevent cycles in stochastic gradient descent. hidden_layers : list (default: [50]) Number of units per hidden layer. By default 50 units in the first hidden layer. At the moment only 1 hidden layer is supported n_classes : int (default: None) A positive integer to declare the number of class labels if not all class labels are present in a partial training set. Gets the number of class labels automatically if None. l1 : float (default: 0.0) L1 regularization strength l2 : float (default: 0.0) L2 regularization strength momentum : float (default: 0.0) Momentum constant. Factor multiplied with the gradient of the previous epoch t-1 to improve learning speed w(t) := w(t) - (grad(t) + momentum * grad(t-1)) decrease_const : float (default: 0.0) Decrease constant. Shrinks the learning rate after each epoch via eta / (1 + epoch*decrease_const) minibatches : int (default: 1) Divide the training data into k minibatches for accelerated stochastic gradient descent learning. Gradient Descent Learning if minibatches = 1 Stochastic Gradient Descent learning if minibatches = len(y) Minibatch learning if minibatches > 1 random_seed : int (default: None) Set random state for shuffling and initializing the weights. print_progress : int (default: 0) Prints progress in fitting to stderr. 0: No output 1: Epochs elapsed and cost 2: 1 plus time elapsed 3: 2 plus estimated time until completion Attributes w_ : 2d-array, shape=[n_features, n_classes] Weights after fitting. b_ : 1D-array, shape=[n_classes] Bias units after fitting. cost_ : list List of floats; the mean categorical cross entropy cost after each epoch. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/classifier/MultiLayerPerceptron/ Methods fit(X, y, init_params=True) Learn model from training data. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values. init_params : bool (default: True) Re-initializes model parameters prior to fitting. Set False to continue training with weights from a previous model fitting. Returns self : object predict(X) Predict targets from X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns target_values : array-like, shape = [n_samples] Predicted target values. predict_proba(X) Predict class probabilities of X from the net input. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns Class probabilties : array-like, shape= [n_samples, n_classes] score(X, y) Compute the prediction accuracy Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values (true class labels). Returns acc : float The prediction accuracy as a float between 0.0 and 1.0 (perfect score). ython","title":"MultilayerPerceptron: A simple multilayer neural network"},{"location":"user_guide/classifier/MultiLayerPerceptron/#multilayerperceptron-a-simple-multilayer-neural-network","text":"Implementation of a multilayer perceptron, a feedforward artificial neural network. from mlxtend.classifier import MultiLayerPerceptron","title":"MultilayerPerceptron: A simple multilayer neural network"},{"location":"user_guide/classifier/MultiLayerPerceptron/#overview","text":"Although the code is fully working and can be used for common classification tasks, this implementation is not geared towards efficiency but clarity \u2013 the original code was written for demonstration purposes.","title":"Overview"},{"location":"user_guide/classifier/MultiLayerPerceptron/#basic-architecture","text":"The neurons x_0 and a_0 represent the bias units ( x_0=1 , a_0=1 ). The i th superscript denotes the i th layer, and the j th subscripts stands for the index of the respective unit. For example, a_{1}^{(2)} refers to the first activation unit after the bias unit (i.e., 2nd activation unit) in the 2nd layer (here: the hidden layer) \\begin{align} \\mathbf{a^{(2)}} &= \\begin{bmatrix} a_{0}^{(2)} \\\\ a_{1}^{(2)} \\\\ \\vdots \\\\ a_{m}^{(2)} \\end{bmatrix}. \\end{align} Each layer (l) in a multi-layer perceptron, a directed graph, is fully connected to the next layer (l+1) . We write the weight coefficient that connects the k th unit in the l th layer to the j th unit in layer l+1 as w^{(l)}_{j, k} . For example, the weight coefficient that connects the units a_0^{(2)} \\rightarrow a_1^{(3)} would be written as w_{1,0}^{(2)} .","title":"Basic Architecture"},{"location":"user_guide/classifier/MultiLayerPerceptron/#activation","text":"In the current implementation, the activations of the hidden layer(s) are computed via the logistic (sigmoid) function \\phi(z) = \\frac{1}{1 + e^{-z}}. (For more details on the logistic function, please see classifier.LogisticRegression ; a general overview of different activation function can be found here .) Furthermore, the MLP uses the softmax function in the output layer, For more details on the logistic function, please see classifier.SoftmaxRegression .","title":"Activation"},{"location":"user_guide/classifier/MultiLayerPerceptron/#references","text":"D. R. G. H. R. Williams and G. Hinton. Learning representations by back-propagating errors . Nature, pages 323\u2013533, 1986. C. M. Bishop. Neural networks for pattern recognition . Oxford University Press, 1995. T. Hastie, J. Friedman, and R. Tibshirani. The Elements of Statistical Learning , Volume 2. Springer, 2009.","title":"References"},{"location":"user_guide/classifier/MultiLayerPerceptron/#example-1-classifying-iris-flowers","text":"Load 2 features from Iris (petal length and petal width) for visualization purposes: from mlxtend.data import iris_data X, y = iris_data() X = X[:, [0, 3]] # standardize training data X_std = (X - X.mean(axis=0)) / X.std(axis=0) Train neural network for 3 output flower classes ('Setosa', 'Versicolor', 'Virginica'), regular gradient decent ( minibatches=1 ), 30 hidden units, and no regularization.","title":"Example 1 - Classifying Iris Flowers"},{"location":"user_guide/classifier/MultiLayerPerceptron/#gradient-descent","text":"Setting the minibatches to 1 will result in gradient descent training; please see Gradient Descent vs. Stochastic Gradient Descent for details. from mlxtend.classifier import MultiLayerPerceptron as MLP nn1 = MLP(hidden_layers=[50], l2=0.00, l1=0.0, epochs=150, eta=0.05, momentum=0.1, decrease_const=0.0, minibatches=1, random_seed=1, print_progress=3) nn1 = nn1.fit(X_std, y) Iteration: 150/150 | Cost 0.06 | Elapsed: 0:00:00 | ETA: 0:00:00 from mlxtend.plotting import plot_decision_regions import matplotlib.pyplot as plt fig = plot_decision_regions(X=X_std, y=y, clf=nn1, legend=2) plt.title('Multi-layer Perceptron w. 1 hidden layer (logistic sigmoid)') plt.show() import matplotlib.pyplot as plt plt.plot(range(len(nn1.cost_)), nn1.cost_) plt.ylabel('Cost') plt.xlabel('Epochs') plt.show() print('Accuracy: %.2f%%' % (100 * nn1.score(X_std, y))) Accuracy: 96.67%","title":"Gradient Descent"},{"location":"user_guide/classifier/MultiLayerPerceptron/#stochastic-gradient-descent","text":"Setting minibatches to n_samples will result in stochastic gradient descent training; please see Gradient Descent vs. Stochastic Gradient Descent for details. nn2 = MLP(hidden_layers=[50], l2=0.00, l1=0.0, epochs=5, eta=0.005, momentum=0.1, decrease_const=0.0, minibatches=len(y), random_seed=1, print_progress=3) nn2.fit(X_std, y) plt.plot(range(len(nn2.cost_)), nn2.cost_) plt.ylabel('Cost') plt.xlabel('Epochs') plt.show() Iteration: 5/5 | Cost 0.11 | Elapsed: 00:00:00 | ETA: 00:00:00 Continue the training for 25 epochs... nn2.epochs = 25 nn2 = nn2.fit(X_std, y) Iteration: 25/25 | Cost 0.07 | Elapsed: 0:00:00 | ETA: 0:00:00 plt.plot(range(len(nn2.cost_)), nn2.cost_) plt.ylabel('Cost') plt.xlabel('Epochs') plt.show()","title":"Stochastic Gradient Descent"},{"location":"user_guide/classifier/MultiLayerPerceptron/#example-2-classifying-handwritten-digits-from-a-10-mnist-subset","text":"Load a 5000-sample subset of the MNIST dataset (please see data.loadlocal_mnist if you want to download and read in the complete MNIST dataset). from mlxtend.data import mnist_data from mlxtend.preprocessing import shuffle_arrays_unison X, y = mnist_data() X, y = shuffle_arrays_unison((X, y), random_seed=1) X_train, y_train = X[:500], y[:500] X_test, y_test = X[500:], y[500:] Visualize a sample from the MNIST dataset to check if it was loaded correctly: import matplotlib.pyplot as plt def plot_digit(X, y, idx): img = X[idx].reshape(28,28) plt.imshow(img, cmap='Greys', interpolation='nearest') plt.title('true label: %d' % y[idx]) plt.show() plot_digit(X, y, 3500) Standardize pixel values: import numpy as np from mlxtend.preprocessing import standardize X_train_std, params = standardize(X_train, columns=range(X_train.shape[1]), return_params=True) X_test_std = standardize(X_test, columns=range(X_test.shape[1]), params=params) Initialize the neural network to recognize the 10 different digits (0-10) using 300 epochs and mini-batch learning. nn1 = MLP(hidden_layers=[150], l2=0.00, l1=0.0, epochs=100, eta=0.005, momentum=0.0, decrease_const=0.0, minibatches=100, random_seed=1, print_progress=3) Learn the features while printing the progress to get an idea about how long it may take. import matplotlib.pyplot as plt nn1.fit(X_train_std, y_train) plt.plot(range(len(nn1.cost_)), nn1.cost_) plt.ylabel('Cost') plt.xlabel('Epochs') plt.show() Iteration: 100/100 | Cost 0.01 | Elapsed: 0:00:17 | ETA: 0:00:00 print('Train Accuracy: %.2f%%' % (100 * nn1.score(X_train_std, y_train))) print('Test Accuracy: %.2f%%' % (100 * nn1.score(X_test_std, y_test))) Train Accuracy: 100.00% Test Accuracy: 84.62% Please note that this neural network has been trained on only 10% of the MNIST data for technical demonstration purposes, hence, the lousy predictive performance.","title":"Example 2 - Classifying Handwritten Digits from a 10% MNIST Subset"},{"location":"user_guide/classifier/MultiLayerPerceptron/#api","text":"MultiLayerPerceptron(eta=0.5, epochs=50, hidden_layers=[50], n_classes=None, momentum=0.0, l1=0.0, l2=0.0, dropout=1.0, decrease_const=0.0, minibatches=1, random_seed=None, print_progress=0) Multi-layer perceptron classifier with logistic sigmoid activations Parameters eta : float (default: 0.5) Learning rate (between 0.0 and 1.0) epochs : int (default: 50) Passes over the training dataset. Prior to each epoch, the dataset is shuffled if minibatches > 1 to prevent cycles in stochastic gradient descent. hidden_layers : list (default: [50]) Number of units per hidden layer. By default 50 units in the first hidden layer. At the moment only 1 hidden layer is supported n_classes : int (default: None) A positive integer to declare the number of class labels if not all class labels are present in a partial training set. Gets the number of class labels automatically if None. l1 : float (default: 0.0) L1 regularization strength l2 : float (default: 0.0) L2 regularization strength momentum : float (default: 0.0) Momentum constant. Factor multiplied with the gradient of the previous epoch t-1 to improve learning speed w(t) := w(t) - (grad(t) + momentum * grad(t-1)) decrease_const : float (default: 0.0) Decrease constant. Shrinks the learning rate after each epoch via eta / (1 + epoch*decrease_const) minibatches : int (default: 1) Divide the training data into k minibatches for accelerated stochastic gradient descent learning. Gradient Descent Learning if minibatches = 1 Stochastic Gradient Descent learning if minibatches = len(y) Minibatch learning if minibatches > 1 random_seed : int (default: None) Set random state for shuffling and initializing the weights. print_progress : int (default: 0) Prints progress in fitting to stderr. 0: No output 1: Epochs elapsed and cost 2: 1 plus time elapsed 3: 2 plus estimated time until completion Attributes w_ : 2d-array, shape=[n_features, n_classes] Weights after fitting. b_ : 1D-array, shape=[n_classes] Bias units after fitting. cost_ : list List of floats; the mean categorical cross entropy cost after each epoch. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/classifier/MultiLayerPerceptron/","title":"API"},{"location":"user_guide/classifier/MultiLayerPerceptron/#methods","text":"fit(X, y, init_params=True) Learn model from training data. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values. init_params : bool (default: True) Re-initializes model parameters prior to fitting. Set False to continue training with weights from a previous model fitting. Returns self : object predict(X) Predict targets from X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns target_values : array-like, shape = [n_samples] Predicted target values. predict_proba(X) Predict class probabilities of X from the net input. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns Class probabilties : array-like, shape= [n_samples, n_classes] score(X, y) Compute the prediction accuracy Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values (true class labels). Returns acc : float The prediction accuracy as a float between 0.0 and 1.0 (perfect score). ython","title":"Methods"},{"location":"user_guide/classifier/OneRClassifier/","text":"OneRClassifier: One Rule (OneR) method for classfication And implementation of the One Rule (OneR) method for classfication. from mlxtend.classifier import OneRClassifier Overview \"OneR\" stands for One Rule (by Robert Holte [1]), which is a classic algorithm for supervised learning. Note that this algorithm is not known for its good prediction performance; thus, it is rather recommended for teaching purposes and for lower-bound performance baselines in real-world applications. The name \"OneRule\" can be a bit misleading, because it is technically about \"one feature\" and not about \"one rule.\" I.e., OneR returns a feature for which one or more decision rules are defined. Essentially, as a simple classifier, it finds exactly one feature (and one or more feature values for that feature) to classify data instances. The basic procedure is as follows: For each feature among all features (columns) in the dataset: For each feature value for the given feature: Obtain the training examples with that feature value. Obtain the class labels (and class label counts) corresponding to the training examples identified in the previous step. Regard the class label with the highest frequency (count) as the majority class. Record the number of errors as the number of training examples that have the given feature value but are not the majority class. Compute the error of the feature by summing the errors for all possible feature values for that feature. Return the best feature, which is defined as the feature with the lowest error. Please note that the OneR algorithm assumes categorical (or discretized) feature values. A nice explanation and OneR classifier can be found in the Interpretable Machine Learning online chapter \"4.5.1 Learn Rules from a Single Feature (OneR)\" (https://christophm.github.io/interpretable-ml-book/rules.html, [2]). References [1] Holte, Robert C. \"Very simple classification rules perform well on most commonly used datasets.\" Machine learning 11.1 (1993): 63-90. [2] Interpretable Machine Learning (2018) by Christoph Molnar: https://christophm.github.io/interpretable-ml-book/rules.html Example 1 -- Demonstrating OneR on a discretized Iris dataset As mentioned in the overview text above, the OneR algorithm expects categorical or discretized features. The OneRClassifier implementation in MLxtend does not modify features in the dataset, and ensuring that the features are categorical is a responsibility of the user. In the following example, we will discretize the Iris dataset. In particular, we will convert the dataset into quartiles. In other words each feature value gets replaced by a categorical value. For sepal width (the first column in Iris), this would be (0, 5.1] => 0 (5.1, 5.8] => 1 (5.8, 6.4] => 2 (6,4, 7.9] => 3 Below are the first 15 rows (flowers) of the original Iris data: from mlxtend.data import iris_data X, y = iris_data() X[:15] array([[5.1, 3.5, 1.4, 0.2], [4.9, 3. , 1.4, 0.2], [4.7, 3.2, 1.3, 0.2], [4.6, 3.1, 1.5, 0.2], [5. , 3.6, 1.4, 0.2], [5.4, 3.9, 1.7, 0.4], [4.6, 3.4, 1.4, 0.3], [5. , 3.4, 1.5, 0.2], [4.4, 2.9, 1.4, 0.2], [4.9, 3.1, 1.5, 0.1], [5.4, 3.7, 1.5, 0.2], [4.8, 3.4, 1.6, 0.2], [4.8, 3. , 1.4, 0.1], [4.3, 3. , 1.1, 0.1], [5.8, 4. , 1.2, 0.2]]) Below is the discretized dataset. Each feature is divided into 4 quartiles. import numpy as np def get_feature_quartiles(X): X_discretized = X.copy() for col in range(X.shape[1]): for q, class_label in zip([1.0, 0.75, 0.5, 0.25], [3, 2, 1, 0]): threshold = np.quantile(X[:, col], q=q) X_discretized[X[:, col] <= threshold, col] = class_label return X_discretized.astype(np.int) Xd = get_feature_quartiles(X) Xd[:15] array([[0, 3, 0, 0], [0, 1, 0, 0], [0, 2, 0, 0], [0, 2, 0, 0], [0, 3, 0, 0], [1, 3, 1, 1], [0, 3, 0, 0], [0, 3, 0, 0], [0, 1, 0, 0], [0, 2, 0, 0], [1, 3, 0, 0], [0, 3, 0, 0], [0, 1, 0, 0], [0, 1, 0, 0], [1, 3, 0, 0]]) Given a dataset with categorical features, we can use the OneR classifier like similar to a scikit-learn estimator for classification. First, let's divide the dataset into training and test data: from sklearn.model_selection import train_test_split Xd_train, Xd_test, y_train, y_test = train_test_split(Xd, y, random_state=0, stratify=y) Next, we can train a OneRClassifier model on the training set using the fit method: from mlxtend.classifier import OneRClassifier oner = OneRClassifier() oner.fit(Xd_train, y_train); The column index of the selected feature is accessible via the feature_idx_ attribute after model fitting: oner.feature_idx_ 2 There is also a prediction_dict_ available after model fitting. It lists the total error for the selected feature (i.e., the feature listed under feature_idx_ ). Also it provides the classification rules: oner.prediction_dict_ {'total error': 16, 'rules (value: class)': {0: 0, 1: 1, 2: 1, 3: 2}} I.e., 'rules (value: class)': {0: 0, 1: 1, 2: 1, 3: 2} means that there are 3 rules for the selected feature (petal length): - if value is 0, classify as 0 (Iris-setosa) - if value is 1, classify as 1 (Iris-versicolor) - if value is 2, classify as 1 (Iris-versicolor) - if value is 3, classify as 2 (Iris-virginica) After model fitting we can use the oner object for making predictions: oner.predict(Xd_train) array([1, 0, 0, 1, 1, 1, 1, 0, 1, 0, 0, 1, 0, 1, 0, 1, 1, 1, 1, 2, 2, 1, 0, 1, 2, 1, 2, 1, 1, 2, 1, 2, 1, 0, 0, 1, 2, 1, 1, 2, 2, 1, 0, 1, 1, 1, 2, 0, 1, 2, 1, 2, 1, 0, 0, 0, 0, 1, 0, 1, 1, 1, 2, 0, 1, 1, 0, 1, 2, 1, 2, 0, 1, 2, 1, 1, 2, 0, 1, 0, 0, 1, 1, 2, 0, 0, 0, 1, 0, 1, 2, 2, 2, 0, 1, 0, 2, 0, 1, 1, 1, 1, 0, 2, 2, 0, 1, 1, 0, 2, 1, 2]) y_pred = oner.predict(Xd_train) train_acc = np.mean(y_pred == y_train) print(f'Training accuracy {train_acc*100:.2f}%') Training accuracy 85.71% y_pred = oner.predict(Xd_test) test_acc = np.mean(y_pred == y_test) print(f'Test accuracy {test_acc*100:.2f}%') Test accuracy 84.21% Instead of computing the prediction accuracy manually as shown above, we can also use the score method: test_acc = oner.score(Xd_test, y_test) print(f'Test accuracy {test_acc*100:.2f}%') Test accuracy 84.21% API OneRClassifier(resolve_ties='first') OneR (One Rule) Classifier. Parameters resolve_ties : str (default: 'first') Option for how to resolve ties if two or more features have the same error. Options are - 'first' (default): chooses first feature in the list, i.e., feature with the lower column index. - 'chi-squared': performs a chi-squared test for each feature against the target and selects the feature with the lowest p-value. Attributes self.classes_labels_ : array-like, shape = [n_labels] Array containing the unique class labels found in the training set. self.feature_idx_ : int The index of the rules' feature based on the column in the training set. self.p_value_ : float The p value for a given feature. Only available after calling fit when the OneR attribute resolve_ties = 'chi-squared' is set. self.prediction_dict_ : dict Dictionary containing information about the feature's (self.feature_idx_) rules and total error. E.g., {'total error': 37, 'rules (value: class)': {0: 0, 1: 2}} means the total error is 37, and the rules are \"if feature value == 0 classify as 0\" and \"if feature value == 1 classify as 2\". (And classify as class 1 otherwise.) For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/classifier/OneRClassifier/ Methods fit(X, y) Learn rule from training data. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values. Returns self : object get_params(deep=True) Get parameters for this estimator. Parameters deep : bool, default=True If True, will return the parameters for this estimator and contained subobjects that are estimators. Returns params : mapping of string to any Parameter names mapped to their values. predict(X) Predict class labels for X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns maj : array-like, shape = [n_samples] Predicted class labels. score(X, y, sample_weight=None) Return the mean accuracy on the given test data and labels. In multi-label classification, this is the subset accuracy which is a harsh metric since you require for each sample that each label set be correctly predicted. Parameters X : array-like of shape (n_samples, n_features) Test samples. y : array-like of shape (n_samples,) or (n_samples, n_outputs) True labels for X. sample_weight : array-like of shape (n_samples,), default=None Sample weights. Returns score : float Mean accuracy of self.predict(X) wrt. y. set_params( params) Set the parameters of this estimator. The method works on simple estimators as well as on nested objects (such as pipelines). The latter have parameters of the form <component>__<parameter> so that it's possible to update each component of a nested object. Parameters **params : dict Estimator parameters. Returns self : object Estimator instance. ython","title":"OneRClassifier: One Rule (OneR) method for classfication"},{"location":"user_guide/classifier/OneRClassifier/#onerclassifier-one-rule-oner-method-for-classfication","text":"And implementation of the One Rule (OneR) method for classfication. from mlxtend.classifier import OneRClassifier","title":"OneRClassifier: One Rule (OneR) method for classfication"},{"location":"user_guide/classifier/OneRClassifier/#overview","text":"\"OneR\" stands for One Rule (by Robert Holte [1]), which is a classic algorithm for supervised learning. Note that this algorithm is not known for its good prediction performance; thus, it is rather recommended for teaching purposes and for lower-bound performance baselines in real-world applications. The name \"OneRule\" can be a bit misleading, because it is technically about \"one feature\" and not about \"one rule.\" I.e., OneR returns a feature for which one or more decision rules are defined. Essentially, as a simple classifier, it finds exactly one feature (and one or more feature values for that feature) to classify data instances. The basic procedure is as follows: For each feature among all features (columns) in the dataset: For each feature value for the given feature: Obtain the training examples with that feature value. Obtain the class labels (and class label counts) corresponding to the training examples identified in the previous step. Regard the class label with the highest frequency (count) as the majority class. Record the number of errors as the number of training examples that have the given feature value but are not the majority class. Compute the error of the feature by summing the errors for all possible feature values for that feature. Return the best feature, which is defined as the feature with the lowest error. Please note that the OneR algorithm assumes categorical (or discretized) feature values. A nice explanation and OneR classifier can be found in the Interpretable Machine Learning online chapter \"4.5.1 Learn Rules from a Single Feature (OneR)\" (https://christophm.github.io/interpretable-ml-book/rules.html, [2]).","title":"Overview"},{"location":"user_guide/classifier/OneRClassifier/#references","text":"[1] Holte, Robert C. \"Very simple classification rules perform well on most commonly used datasets.\" Machine learning 11.1 (1993): 63-90. [2] Interpretable Machine Learning (2018) by Christoph Molnar: https://christophm.github.io/interpretable-ml-book/rules.html","title":"References"},{"location":"user_guide/classifier/OneRClassifier/#example-1-demonstrating-oner-on-a-discretized-iris-dataset","text":"As mentioned in the overview text above, the OneR algorithm expects categorical or discretized features. The OneRClassifier implementation in MLxtend does not modify features in the dataset, and ensuring that the features are categorical is a responsibility of the user. In the following example, we will discretize the Iris dataset. In particular, we will convert the dataset into quartiles. In other words each feature value gets replaced by a categorical value. For sepal width (the first column in Iris), this would be (0, 5.1] => 0 (5.1, 5.8] => 1 (5.8, 6.4] => 2 (6,4, 7.9] => 3 Below are the first 15 rows (flowers) of the original Iris data: from mlxtend.data import iris_data X, y = iris_data() X[:15] array([[5.1, 3.5, 1.4, 0.2], [4.9, 3. , 1.4, 0.2], [4.7, 3.2, 1.3, 0.2], [4.6, 3.1, 1.5, 0.2], [5. , 3.6, 1.4, 0.2], [5.4, 3.9, 1.7, 0.4], [4.6, 3.4, 1.4, 0.3], [5. , 3.4, 1.5, 0.2], [4.4, 2.9, 1.4, 0.2], [4.9, 3.1, 1.5, 0.1], [5.4, 3.7, 1.5, 0.2], [4.8, 3.4, 1.6, 0.2], [4.8, 3. , 1.4, 0.1], [4.3, 3. , 1.1, 0.1], [5.8, 4. , 1.2, 0.2]]) Below is the discretized dataset. Each feature is divided into 4 quartiles. import numpy as np def get_feature_quartiles(X): X_discretized = X.copy() for col in range(X.shape[1]): for q, class_label in zip([1.0, 0.75, 0.5, 0.25], [3, 2, 1, 0]): threshold = np.quantile(X[:, col], q=q) X_discretized[X[:, col] <= threshold, col] = class_label return X_discretized.astype(np.int) Xd = get_feature_quartiles(X) Xd[:15] array([[0, 3, 0, 0], [0, 1, 0, 0], [0, 2, 0, 0], [0, 2, 0, 0], [0, 3, 0, 0], [1, 3, 1, 1], [0, 3, 0, 0], [0, 3, 0, 0], [0, 1, 0, 0], [0, 2, 0, 0], [1, 3, 0, 0], [0, 3, 0, 0], [0, 1, 0, 0], [0, 1, 0, 0], [1, 3, 0, 0]]) Given a dataset with categorical features, we can use the OneR classifier like similar to a scikit-learn estimator for classification. First, let's divide the dataset into training and test data: from sklearn.model_selection import train_test_split Xd_train, Xd_test, y_train, y_test = train_test_split(Xd, y, random_state=0, stratify=y) Next, we can train a OneRClassifier model on the training set using the fit method: from mlxtend.classifier import OneRClassifier oner = OneRClassifier() oner.fit(Xd_train, y_train); The column index of the selected feature is accessible via the feature_idx_ attribute after model fitting: oner.feature_idx_ 2 There is also a prediction_dict_ available after model fitting. It lists the total error for the selected feature (i.e., the feature listed under feature_idx_ ). Also it provides the classification rules: oner.prediction_dict_ {'total error': 16, 'rules (value: class)': {0: 0, 1: 1, 2: 1, 3: 2}} I.e., 'rules (value: class)': {0: 0, 1: 1, 2: 1, 3: 2} means that there are 3 rules for the selected feature (petal length): - if value is 0, classify as 0 (Iris-setosa) - if value is 1, classify as 1 (Iris-versicolor) - if value is 2, classify as 1 (Iris-versicolor) - if value is 3, classify as 2 (Iris-virginica) After model fitting we can use the oner object for making predictions: oner.predict(Xd_train) array([1, 0, 0, 1, 1, 1, 1, 0, 1, 0, 0, 1, 0, 1, 0, 1, 1, 1, 1, 2, 2, 1, 0, 1, 2, 1, 2, 1, 1, 2, 1, 2, 1, 0, 0, 1, 2, 1, 1, 2, 2, 1, 0, 1, 1, 1, 2, 0, 1, 2, 1, 2, 1, 0, 0, 0, 0, 1, 0, 1, 1, 1, 2, 0, 1, 1, 0, 1, 2, 1, 2, 0, 1, 2, 1, 1, 2, 0, 1, 0, 0, 1, 1, 2, 0, 0, 0, 1, 0, 1, 2, 2, 2, 0, 1, 0, 2, 0, 1, 1, 1, 1, 0, 2, 2, 0, 1, 1, 0, 2, 1, 2]) y_pred = oner.predict(Xd_train) train_acc = np.mean(y_pred == y_train) print(f'Training accuracy {train_acc*100:.2f}%') Training accuracy 85.71% y_pred = oner.predict(Xd_test) test_acc = np.mean(y_pred == y_test) print(f'Test accuracy {test_acc*100:.2f}%') Test accuracy 84.21% Instead of computing the prediction accuracy manually as shown above, we can also use the score method: test_acc = oner.score(Xd_test, y_test) print(f'Test accuracy {test_acc*100:.2f}%') Test accuracy 84.21%","title":"Example 1 -- Demonstrating OneR on a discretized Iris dataset"},{"location":"user_guide/classifier/OneRClassifier/#api","text":"OneRClassifier(resolve_ties='first') OneR (One Rule) Classifier. Parameters resolve_ties : str (default: 'first') Option for how to resolve ties if two or more features have the same error. Options are - 'first' (default): chooses first feature in the list, i.e., feature with the lower column index. - 'chi-squared': performs a chi-squared test for each feature against the target and selects the feature with the lowest p-value. Attributes self.classes_labels_ : array-like, shape = [n_labels] Array containing the unique class labels found in the training set. self.feature_idx_ : int The index of the rules' feature based on the column in the training set. self.p_value_ : float The p value for a given feature. Only available after calling fit when the OneR attribute resolve_ties = 'chi-squared' is set. self.prediction_dict_ : dict Dictionary containing information about the feature's (self.feature_idx_) rules and total error. E.g., {'total error': 37, 'rules (value: class)': {0: 0, 1: 2}} means the total error is 37, and the rules are \"if feature value == 0 classify as 0\" and \"if feature value == 1 classify as 2\". (And classify as class 1 otherwise.) For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/classifier/OneRClassifier/","title":"API"},{"location":"user_guide/classifier/OneRClassifier/#methods","text":"fit(X, y) Learn rule from training data. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values. Returns self : object get_params(deep=True) Get parameters for this estimator. Parameters deep : bool, default=True If True, will return the parameters for this estimator and contained subobjects that are estimators. Returns params : mapping of string to any Parameter names mapped to their values. predict(X) Predict class labels for X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns maj : array-like, shape = [n_samples] Predicted class labels. score(X, y, sample_weight=None) Return the mean accuracy on the given test data and labels. In multi-label classification, this is the subset accuracy which is a harsh metric since you require for each sample that each label set be correctly predicted. Parameters X : array-like of shape (n_samples, n_features) Test samples. y : array-like of shape (n_samples,) or (n_samples, n_outputs) True labels for X. sample_weight : array-like of shape (n_samples,), default=None Sample weights. Returns score : float Mean accuracy of self.predict(X) wrt. y. set_params( params) Set the parameters of this estimator. The method works on simple estimators as well as on nested objects (such as pipelines). The latter have parameters of the form <component>__<parameter> so that it's possible to update each component of a nested object. Parameters **params : dict Estimator parameters. Returns self : object Estimator instance. ython","title":"Methods"},{"location":"user_guide/classifier/Perceptron/","text":"Perceptron: A simple binary classifier Implementation of a Perceptron learning algorithm for classification. from mlxtend.classifier import Perceptron Overview The idea behind this \"thresholded\" perceptron was to mimic how a single neuron in the brain works: It either \"fires\" or not. A perceptron receives multiple input signals, and if the sum of the input signals exceed a certain threshold it either returns a signal or remains \"silent\" otherwise. What made this a \"machine learning\" algorithm was Frank Rosenblatt's idea of the perceptron learning rule: The perceptron algorithm is about learning the weights for the input signals in order to draw linear decision boundary that allows us to discriminate between the two linearly separable classes +1 and -1. Basic Notation Before we dive deeper into the algorithm(s) for learning the weights of the perceptron classifier, let us take a brief look at the basic notation. In the following sections, we will label the positive and negative class in our binary classification setting as \"1\" and \"-1\", respectively. Next, we define an activation function g(\\mathbf{z}) that takes a linear combination of the input values \\mathbf{x} and weights \\mathbf{w} as input ( \\mathbf{z} = w_1x_{1} + \\dots + w_mx_{m} ), and if g(\\mathbf{z}) is greater than a defined threshold \\theta we predict 1 and -1 otherwise; in this case, this activation function g is a simple \"unit step function,\" which is sometimes also called \"Heaviside step function.\" $$ g(z) = \\begin{cases} 1 & \\text{if $z \\ge \\theta$}\\\\ -1 & \\text{otherwise}. \\end{cases} $$ where z = w_1x_{1} + \\dots + w_mx_{m} = \\sum_{j=1}^{m} x_{j}w_{j} \\\\ = \\mathbf{w}^T\\mathbf{x} \\mathbf{w} is the feature vector, and \\mathbf{x} is an m -dimensional sample from the training dataset: \\mathbf{w} = \\begin{bmatrix} w_{1} \\\\ \\vdots \\\\ w_{m} \\end{bmatrix} \\quad \\mathbf{x} = \\begin{bmatrix} x_{1} \\\\ \\vdots \\\\ x_{m} \\end{bmatrix} In order to simplify the notation, we bring \\theta to the left side of the equation and define w_0 = -\\theta \\text{ and } x_0=1 so that $$ g({z}) = \\begin{cases} 1 & \\text{if $z \\ge 0$}\\\\ -1 & \\text{otherwise}. \\end{cases} $$ and z = w_0x_{0} + w_1x_{1} + \\dots + w_mx_{m} = \\sum_{j=0}^{m} x_{j}w_{j} \\\\ = \\mathbf{w}^T\\mathbf{x}. Perceptron Rule Rosenblatt's initial perceptron rule is fairly simple and can be summarized by the following steps: Initialize the weights to 0 or small random numbers. For each training sample \\mathbf{x^{(i)}} : Calculate the output value. Update the weights. The output value is the class label predicted by the unit step function that we defined earlier (output =g(\\mathbf{z}) ) and the weight update can be written more formally as w_j := w_j + \\Delta w_j . The value for updating the weights at each increment is calculated by the learning rule \\Delta w_j = \\eta \\; (\\text{target}^{(i)} - \\text{output}^{(i)})\\;x^{(i)}_{j} where \\eta is the learning rate (a constant between 0.0 and 1.0), \"target\" is the true class label, and the \"output\" is the predicted class label. aIt is important to note that all weights in the weight vector are being updated simultaneously. Concretely, for a 2-dimensional dataset, we would write the update as: \\Delta w_0 = \\eta(\\text{target}^{(i)} - \\text{output}^{(i)}) \\Delta w_1 = \\eta(\\text{target}^{(i)} - \\text{output}^{(i)})\\;x^{(i)}_{1} \\Delta w_2 = \\eta(\\text{target}^{(i)} - \\text{output}^{(i)})\\;x^{(i)}_{2} Before we implement the perceptron rule in Python, let us make a simple thought experiment to illustrate how beautifully simple this learning rule really is. In the two scenarios where the perceptron predicts the class label correctly, the weights remain unchanged: \\Delta w_j = \\eta(-1^{(i)} - -1^{(i)})\\;x^{(i)}_{j} = 0 \\Delta w_j = \\eta(1^{(i)} - 1^{(i)})\\;x^{(i)}_{j} = 0 However, in case of a wrong prediction, the weights are being \"pushed\" towards the direction of the positive or negative target class, respectively: \\Delta w_j = \\eta(1^{(i)} - -1^{(i)})\\;x^{(i)}_{j} = \\eta(2)\\;x^{(i)}_{j} \\Delta w_j = \\eta(-1^{(i)} - 1^{(i)})\\;x^{(i)}_{j} = \\eta(-2)\\;x^{(i)}_{j} It is important to note that the convergence of the perceptron is only guaranteed if the two classes are linearly separable. If the two classes can't be separated by a linear decision boundary, we can set a maximum number of passes over the training dataset (\"epochs\") and/or a threshold for the number of tolerated misclassifications. References F. Rosenblatt. The perceptron, a perceiving and recognizing automaton Project Para. Cornell Aeronautical Laboratory, 1957. Example 1 - Classification of Iris Flowers from mlxtend.data import iris_data from mlxtend.plotting import plot_decision_regions from mlxtend.classifier import Perceptron import matplotlib.pyplot as plt # Loading Data X, y = iris_data() X = X[:, [0, 3]] # sepal length and petal width X = X[0:100] # class 0 and class 1 y = y[0:100] # class 0 and class 1 # standardize X[:,0] = (X[:,0] - X[:,0].mean()) / X[:,0].std() X[:,1] = (X[:,1] - X[:,1].mean()) / X[:,1].std() # Rosenblatt Perceptron ppn = Perceptron(epochs=5, eta=0.05, random_seed=0, print_progress=3) ppn.fit(X, y) plot_decision_regions(X, y, clf=ppn) plt.title('Perceptron - Rosenblatt Perceptron Rule') plt.show() print('Bias & Weights: %s' % ppn.w_) plt.plot(range(len(ppn.cost_)), ppn.cost_) plt.xlabel('Iterations') plt.ylabel('Missclassifications') plt.show() Iteration: 5/5 | Elapsed: 00:00:00 | ETA: 00:00:00 Bias & Weights: [[-0.04500809] [ 0.11048855]] API Perceptron(eta=0.1, epochs=50, random_seed=None, print_progress=0) Perceptron classifier. Note that this implementation of the Perceptron expects binary class labels in {0, 1}. Parameters eta : float (default: 0.1) Learning rate (between 0.0 and 1.0) epochs : int (default: 50) Number of passes over the training dataset. Prior to each epoch, the dataset is shuffled to prevent cycles. random_seed : int Random state for initializing random weights and shuffling. print_progress : int (default: 0) Prints progress in fitting to stderr. 0: No output 1: Epochs elapsed and cost 2: 1 plus time elapsed 3: 2 plus estimated time until completion Attributes w_ : 2d-array, shape={n_features, 1} Model weights after fitting. b_ : 1d-array, shape={1,} Bias unit after fitting. cost_ : list Number of misclassifications in every epoch. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/classifier/Perceptron/ Methods fit(X, y, init_params=True) Learn model from training data. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values. init_params : bool (default: True) Re-initializes model parameters prior to fitting. Set False to continue training with weights from a previous model fitting. Returns self : object predict(X) Predict targets from X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns target_values : array-like, shape = [n_samples] Predicted target values. score(X, y) Compute the prediction accuracy Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values (true class labels). Returns acc : float The prediction accuracy as a float between 0.0 and 1.0 (perfect score). ython","title":"Perceptron: A simple binary classifier"},{"location":"user_guide/classifier/Perceptron/#perceptron-a-simple-binary-classifier","text":"Implementation of a Perceptron learning algorithm for classification. from mlxtend.classifier import Perceptron","title":"Perceptron: A simple binary classifier"},{"location":"user_guide/classifier/Perceptron/#overview","text":"The idea behind this \"thresholded\" perceptron was to mimic how a single neuron in the brain works: It either \"fires\" or not. A perceptron receives multiple input signals, and if the sum of the input signals exceed a certain threshold it either returns a signal or remains \"silent\" otherwise. What made this a \"machine learning\" algorithm was Frank Rosenblatt's idea of the perceptron learning rule: The perceptron algorithm is about learning the weights for the input signals in order to draw linear decision boundary that allows us to discriminate between the two linearly separable classes +1 and -1.","title":"Overview"},{"location":"user_guide/classifier/Perceptron/#basic-notation","text":"Before we dive deeper into the algorithm(s) for learning the weights of the perceptron classifier, let us take a brief look at the basic notation. In the following sections, we will label the positive and negative class in our binary classification setting as \"1\" and \"-1\", respectively. Next, we define an activation function g(\\mathbf{z}) that takes a linear combination of the input values \\mathbf{x} and weights \\mathbf{w} as input ( \\mathbf{z} = w_1x_{1} + \\dots + w_mx_{m} ), and if g(\\mathbf{z}) is greater than a defined threshold \\theta we predict 1 and -1 otherwise; in this case, this activation function g is a simple \"unit step function,\" which is sometimes also called \"Heaviside step function.\" $$ g(z) = \\begin{cases} 1 & \\text{if $z \\ge \\theta$}\\\\ -1 & \\text{otherwise}. \\end{cases} $$ where z = w_1x_{1} + \\dots + w_mx_{m} = \\sum_{j=1}^{m} x_{j}w_{j} \\\\ = \\mathbf{w}^T\\mathbf{x} \\mathbf{w} is the feature vector, and \\mathbf{x} is an m -dimensional sample from the training dataset: \\mathbf{w} = \\begin{bmatrix} w_{1} \\\\ \\vdots \\\\ w_{m} \\end{bmatrix} \\quad \\mathbf{x} = \\begin{bmatrix} x_{1} \\\\ \\vdots \\\\ x_{m} \\end{bmatrix} In order to simplify the notation, we bring \\theta to the left side of the equation and define w_0 = -\\theta \\text{ and } x_0=1 so that $$ g({z}) = \\begin{cases} 1 & \\text{if $z \\ge 0$}\\\\ -1 & \\text{otherwise}. \\end{cases} $$ and z = w_0x_{0} + w_1x_{1} + \\dots + w_mx_{m} = \\sum_{j=0}^{m} x_{j}w_{j} \\\\ = \\mathbf{w}^T\\mathbf{x}.","title":"Basic Notation"},{"location":"user_guide/classifier/Perceptron/#perceptron-rule","text":"Rosenblatt's initial perceptron rule is fairly simple and can be summarized by the following steps: Initialize the weights to 0 or small random numbers. For each training sample \\mathbf{x^{(i)}} : Calculate the output value. Update the weights. The output value is the class label predicted by the unit step function that we defined earlier (output =g(\\mathbf{z}) ) and the weight update can be written more formally as w_j := w_j + \\Delta w_j . The value for updating the weights at each increment is calculated by the learning rule \\Delta w_j = \\eta \\; (\\text{target}^{(i)} - \\text{output}^{(i)})\\;x^{(i)}_{j} where \\eta is the learning rate (a constant between 0.0 and 1.0), \"target\" is the true class label, and the \"output\" is the predicted class label. aIt is important to note that all weights in the weight vector are being updated simultaneously. Concretely, for a 2-dimensional dataset, we would write the update as: \\Delta w_0 = \\eta(\\text{target}^{(i)} - \\text{output}^{(i)}) \\Delta w_1 = \\eta(\\text{target}^{(i)} - \\text{output}^{(i)})\\;x^{(i)}_{1} \\Delta w_2 = \\eta(\\text{target}^{(i)} - \\text{output}^{(i)})\\;x^{(i)}_{2} Before we implement the perceptron rule in Python, let us make a simple thought experiment to illustrate how beautifully simple this learning rule really is. In the two scenarios where the perceptron predicts the class label correctly, the weights remain unchanged: \\Delta w_j = \\eta(-1^{(i)} - -1^{(i)})\\;x^{(i)}_{j} = 0 \\Delta w_j = \\eta(1^{(i)} - 1^{(i)})\\;x^{(i)}_{j} = 0 However, in case of a wrong prediction, the weights are being \"pushed\" towards the direction of the positive or negative target class, respectively: \\Delta w_j = \\eta(1^{(i)} - -1^{(i)})\\;x^{(i)}_{j} = \\eta(2)\\;x^{(i)}_{j} \\Delta w_j = \\eta(-1^{(i)} - 1^{(i)})\\;x^{(i)}_{j} = \\eta(-2)\\;x^{(i)}_{j} It is important to note that the convergence of the perceptron is only guaranteed if the two classes are linearly separable. If the two classes can't be separated by a linear decision boundary, we can set a maximum number of passes over the training dataset (\"epochs\") and/or a threshold for the number of tolerated misclassifications.","title":"Perceptron Rule"},{"location":"user_guide/classifier/Perceptron/#references","text":"F. Rosenblatt. The perceptron, a perceiving and recognizing automaton Project Para. Cornell Aeronautical Laboratory, 1957.","title":"References"},{"location":"user_guide/classifier/Perceptron/#example-1-classification-of-iris-flowers","text":"from mlxtend.data import iris_data from mlxtend.plotting import plot_decision_regions from mlxtend.classifier import Perceptron import matplotlib.pyplot as plt # Loading Data X, y = iris_data() X = X[:, [0, 3]] # sepal length and petal width X = X[0:100] # class 0 and class 1 y = y[0:100] # class 0 and class 1 # standardize X[:,0] = (X[:,0] - X[:,0].mean()) / X[:,0].std() X[:,1] = (X[:,1] - X[:,1].mean()) / X[:,1].std() # Rosenblatt Perceptron ppn = Perceptron(epochs=5, eta=0.05, random_seed=0, print_progress=3) ppn.fit(X, y) plot_decision_regions(X, y, clf=ppn) plt.title('Perceptron - Rosenblatt Perceptron Rule') plt.show() print('Bias & Weights: %s' % ppn.w_) plt.plot(range(len(ppn.cost_)), ppn.cost_) plt.xlabel('Iterations') plt.ylabel('Missclassifications') plt.show() Iteration: 5/5 | Elapsed: 00:00:00 | ETA: 00:00:00 Bias & Weights: [[-0.04500809] [ 0.11048855]]","title":"Example 1 - Classification of Iris Flowers"},{"location":"user_guide/classifier/Perceptron/#api","text":"Perceptron(eta=0.1, epochs=50, random_seed=None, print_progress=0) Perceptron classifier. Note that this implementation of the Perceptron expects binary class labels in {0, 1}. Parameters eta : float (default: 0.1) Learning rate (between 0.0 and 1.0) epochs : int (default: 50) Number of passes over the training dataset. Prior to each epoch, the dataset is shuffled to prevent cycles. random_seed : int Random state for initializing random weights and shuffling. print_progress : int (default: 0) Prints progress in fitting to stderr. 0: No output 1: Epochs elapsed and cost 2: 1 plus time elapsed 3: 2 plus estimated time until completion Attributes w_ : 2d-array, shape={n_features, 1} Model weights after fitting. b_ : 1d-array, shape={1,} Bias unit after fitting. cost_ : list Number of misclassifications in every epoch. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/classifier/Perceptron/","title":"API"},{"location":"user_guide/classifier/Perceptron/#methods","text":"fit(X, y, init_params=True) Learn model from training data. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values. init_params : bool (default: True) Re-initializes model parameters prior to fitting. Set False to continue training with weights from a previous model fitting. Returns self : object predict(X) Predict targets from X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns target_values : array-like, shape = [n_samples] Predicted target values. score(X, y) Compute the prediction accuracy Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values (true class labels). Returns acc : float The prediction accuracy as a float between 0.0 and 1.0 (perfect score). ython","title":"Methods"},{"location":"user_guide/classifier/SoftmaxRegression/","text":"SoftmaxRegression: Multiclass version of logistic regression A logistic regression class for multi-class classification tasks. from mlxtend.classifier import SoftmaxRegression Overview Softmax Regression (synonyms: Multinomial Logistic , Maximum Entropy Classifier , or just Multi-class Logistic Regression ) is a generalization of logistic regression that we can use for multi-class classification (under the assumption that the classes are mutually exclusive). In contrast, we use the (standard) Logistic Regression model in binary classification tasks. Below is a schematic of a Logistic Regression model, for more details, please see the LogisticRegression manual . In Softmax Regression (SMR), we replace the sigmoid logistic function by the so-called softmax function \\phi_{softmax}(\\cdot) . P(y=j \\mid z^{(i)}) = \\phi_{softmax}(z^{(i)}) = \\frac{e^{z^{(i)}}}{\\sum_{j=0}^{k} e^{z_{k}^{(i)}}}, where we define the net input z as z = w_1x_1 + ... + w_mx_m + b= \\sum_{l=1}^{m} w_l x_l + b= \\mathbf{w}^T\\mathbf{x} + b. ( w is the weight vector, \\mathbf{x} is the feature vector of 1 training sample, and b is the bias unit.) Now, this softmax function computes the probability that this training sample \\mathbf{x}^{(i)} belongs to class j given the weight and net input z^{(i)} . So, we compute the probability p(y = j \\mid \\mathbf{x^{(i)}; w}_j) for each class label in j = 1, \\ldots, k. . Note the normalization term in the denominator which causes these class probabilities to sum up to one. To illustrate the concept of softmax, let us walk through a concrete example. Let's assume we have a training set consisting of 4 samples from 3 different classes (0, 1, and 2) x_0 \\rightarrow \\text{class }0 x_1 \\rightarrow \\text{class }1 x_2 \\rightarrow \\text{class }2 x_3 \\rightarrow \\text{class }2 import numpy as np y = np.array([0, 1, 2, 2]) First, we want to encode the class labels into a format that we can more easily work with; we apply one-hot encoding: y_enc = (np.arange(np.max(y) + 1) == y[:, None]).astype(float) print('one-hot encoding:\\n', y_enc) one-hot encoding: [[ 1. 0. 0.] [ 0. 1. 0.] [ 0. 0. 1.] [ 0. 0. 1.]] A sample that belongs to class 0 (the first row) has a 1 in the first cell, a sample that belongs to class 2 has a 1 in the second cell of its row, and so forth. Next, let us define the feature matrix of our 4 training samples. Here, we assume that our dataset consists of 2 features; thus, we create a 4x2 dimensional matrix of our samples and features. Similarly, we create a 2x3 dimensional weight matrix (one row per feature and one column for each class). X = np.array([[0.1, 0.5], [1.1, 2.3], [-1.1, -2.3], [-1.5, -2.5]]) W = np.array([[0.1, 0.2, 0.3], [0.1, 0.2, 0.3]]) bias = np.array([0.01, 0.1, 0.1]) print('Inputs X:\\n', X) print('\\nWeights W:\\n', W) print('\\nbias:\\n', bias) Inputs X: [[ 0.1 0.5] [ 1.1 2.3] [-1.1 -2.3] [-1.5 -2.5]] Weights W: [[ 0.1 0.2 0.3] [ 0.1 0.2 0.3]] bias: [ 0.01 0.1 0.1 ] To compute the net input, we multiply the 4x2 matrix feature matrix X with the 2x3 (n_features x n_classes) weight matrix W , which yields a 4x3 output matrix (n_samples x n_classes) to which we then add the bias unit: \\mathbf{Z} = \\mathbf{X}\\mathbf{W} + \\mathbf{b}. X = np.array([[0.1, 0.5], [1.1, 2.3], [-1.1, -2.3], [-1.5, -2.5]]) W = np.array([[0.1, 0.2, 0.3], [0.1, 0.2, 0.3]]) bias = np.array([0.01, 0.1, 0.1]) print('Inputs X:\\n', X) print('\\nWeights W:\\n', W) print('\\nbias:\\n', bias) Inputs X: [[ 0.1 0.5] [ 1.1 2.3] [-1.1 -2.3] [-1.5 -2.5]] Weights W: [[ 0.1 0.2 0.3] [ 0.1 0.2 0.3]] bias: [ 0.01 0.1 0.1 ] def net_input(X, W, b): return (X.dot(W) + b) net_in = net_input(X, W, bias) print('net input:\\n', net_in) net input: [[ 0.07 0.22 0.28] [ 0.35 0.78 1.12] [-0.33 -0.58 -0.92] [-0.39 -0.7 -1.1 ]] Now, it's time to compute the softmax activation that we discussed earlier: P(y=j \\mid z^{(i)}) = \\phi_{softmax}(z^{(i)}) = \\frac{e^{z^{(i)}}}{\\sum_{j=0}^{k} e^{z_{k}^{(i)}}}. def softmax(z): return (np.exp(z.T) / np.sum(np.exp(z), axis=1)).T smax = softmax(net_in) print('softmax:\\n', smax) softmax: [[ 0.29450637 0.34216758 0.36332605] [ 0.21290077 0.32728332 0.45981591] [ 0.42860913 0.33380113 0.23758974] [ 0.44941979 0.32962558 0.22095463]] As we can see, the values for each sample (row) nicely sum up to 1 now. E.g., we can say that the first sample [ 0.29450637 0.34216758 0.36332605] has a 29.45% probability to belong to class 0. Now, in order to turn these probabilities back into class labels, we could simply take the argmax-index position of each row: [[ 0.29450637 0.34216758 0.36332605 ] -> 2 [ 0.21290077 0.32728332 0.45981591 ] -> 2 [ 0.42860913 0.33380113 0.23758974] -> 0 [ 0.44941979 0.32962558 0.22095463]] -> 0 def to_classlabel(z): return z.argmax(axis=1) print('predicted class labels: ', to_classlabel(smax)) predicted class labels: [2 2 0 0] As we can see, our predictions are terribly wrong, since the correct class labels are [0, 1, 2, 2] . Now, in order to train our logistic model (e.g., via an optimization algorithm such as gradient descent), we need to define a cost function J(\\cdot) that we want to minimize: J(\\mathbf{W}; \\mathbf{b}) = \\frac{1}{n} \\sum_{i=1}^{n} H(T_i, O_i), which is the average of all cross-entropies over our n training samples. The cross-entropy function is defined as H(T_i, O_i) = -\\sum_m T_i \\cdot log(O_i). Here the T stands for \"target\" (i.e., the true class labels) and the O stands for output -- the computed probability via softmax; not the predicted class label. def cross_entropy(output, y_target): return - np.sum(np.log(output) * (y_target), axis=1) xent = cross_entropy(smax, y_enc) print('Cross Entropy:', xent) Cross Entropy: [ 1.22245465 1.11692907 1.43720989 1.50979788] def cost(output, y_target): return np.mean(cross_entropy(output, y_target)) J_cost = cost(smax, y_enc) print('Cost: ', J_cost) Cost: 1.32159787159 In order to learn our softmax model -- determining the weight coefficients -- via gradient descent, we then need to compute the derivative \\nabla \\mathbf{w}_j \\, J(\\mathbf{W}; \\mathbf{b}). I don't want to walk through the tedious details here, but this cost derivative turns out to be simply: \\nabla \\mathbf{w}_j \\, J(\\mathbf{W}; \\mathbf{b}) = \\frac{1}{n} \\sum^{n}_{i=0} \\big[\\mathbf{x}^{(i)}\\ \\big(O_i - T_i \\big) \\big] We can then use the cost derivate to update the weights in opposite direction of the cost gradient with learning rate \\eta : \\mathbf{w}_j := \\mathbf{w}_j - \\eta \\nabla \\mathbf{w}_j \\, J(\\mathbf{W}; \\mathbf{b}) for each class j \\in \\{0, 1, ..., k\\} (note that \\mathbf{w}_j is the weight vector for the class y=j ), and we update the bias units \\mathbf{b}_j := \\mathbf{b}_j - \\eta \\bigg[ \\frac{1}{n} \\sum^{n}_{i=0} \\big(O_i - T_i \\big) \\bigg]. As a penalty against complexity, an approach to reduce the variance of our model and decrease the degree of overfitting by adding additional bias, we can further add a regularization term such as the L2 term with the regularization parameter \\lambda : L2: \\frac{\\lambda}{2} ||\\mathbf{w}||_{2}^{2} , where ||\\mathbf{w}||_{2}^{2} = \\sum^{m}_{l=0} \\sum^{k}_{j=0} w_{i, j} so that our cost function becomes J(\\mathbf{W}; \\mathbf{b}) = \\frac{1}{n} \\sum_{i=1}^{n} H(T_i, O_i) + \\frac{\\lambda}{2} ||\\mathbf{w}||_{2}^{2} and we define the \"regularized\" weight update as \\mathbf{w}_j := \\mathbf{w}_j - \\eta \\big[\\nabla \\mathbf{w}_j \\, J(\\mathbf{W}) + \\lambda \\mathbf{w}_j \\big]. (Please note that we don't regularize the bias term.) Example 1 - Gradient Descent from mlxtend.data import iris_data from mlxtend.plotting import plot_decision_regions from mlxtend.classifier import SoftmaxRegression import matplotlib.pyplot as plt # Loading Data X, y = iris_data() X = X[:, [0, 3]] # sepal length and petal width # standardize X[:,0] = (X[:,0] - X[:,0].mean()) / X[:,0].std() X[:,1] = (X[:,1] - X[:,1].mean()) / X[:,1].std() lr = SoftmaxRegression(eta=0.01, epochs=500, minibatches=1, random_seed=1, print_progress=3) lr.fit(X, y) plot_decision_regions(X, y, clf=lr) plt.title('Softmax Regression - Gradient Descent') plt.show() plt.plot(range(len(lr.cost_)), lr.cost_) plt.xlabel('Iterations') plt.ylabel('Cost') plt.show() Iteration: 500/500 | Cost 0.06 | Elapsed: 0:00:00 | ETA: 0:00:00 Predicting Class Labels y_pred = lr.predict(X) print('Last 3 Class Labels: %s' % y_pred[-3:]) Last 3 Class Labels: [2 2 2] Predicting Class Probabilities y_pred = lr.predict_proba(X) print('Last 3 Class Labels:\\n %s' % y_pred[-3:]) Last 3 Class Labels: [[ 9.18728149e-09 1.68894679e-02 9.83110523e-01] [ 2.97052325e-11 7.26356627e-04 9.99273643e-01] [ 1.57464093e-06 1.57779528e-01 8.42218897e-01]] Example 2 - Stochastic Gradient Descent from mlxtend.data import iris_data from mlxtend.plotting import plot_decision_regions from mlxtend.classifier import SoftmaxRegression import matplotlib.pyplot as plt # Loading Data X, y = iris_data() X = X[:, [0, 3]] # sepal length and petal width # standardize X[:,0] = (X[:,0] - X[:,0].mean()) / X[:,0].std() X[:,1] = (X[:,1] - X[:,1].mean()) / X[:,1].std() lr = SoftmaxRegression(eta=0.01, epochs=300, minibatches=len(y), random_seed=1) lr.fit(X, y) plot_decision_regions(X, y, clf=lr) plt.title('Softmax Regression - Stochastic Gradient Descent') plt.show() plt.plot(range(len(lr.cost_)), lr.cost_) plt.xlabel('Iterations') plt.ylabel('Cost') plt.show() API SoftmaxRegression(eta=0.01, epochs=50, l2=0.0, minibatches=1, n_classes=None, random_seed=None, print_progress=0) Softmax regression classifier. Parameters eta : float (default: 0.01) Learning rate (between 0.0 and 1.0) epochs : int (default: 50) Passes over the training dataset. Prior to each epoch, the dataset is shuffled if minibatches > 1 to prevent cycles in stochastic gradient descent. l2 : float Regularization parameter for L2 regularization. No regularization if l2=0.0. minibatches : int (default: 1) The number of minibatches for gradient-based optimization. If 1: Gradient Descent learning If len(y): Stochastic Gradient Descent (SGD) online learning If 1 < minibatches < len(y): SGD Minibatch learning n_classes : int (default: None) A positive integer to declare the number of class labels if not all class labels are present in a partial training set. Gets the number of class labels automatically if None. random_seed : int (default: None) Set random state for shuffling and initializing the weights. print_progress : int (default: 0) Prints progress in fitting to stderr. 0: No output 1: Epochs elapsed and cost 2: 1 plus time elapsed 3: 2 plus estimated time until completion Attributes w_ : 2d-array, shape={n_features, 1} Model weights after fitting. b_ : 1d-array, shape={1,} Bias unit after fitting. cost_ : list List of floats, the average cross_entropy for each epoch. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/classifier/SoftmaxRegression/ Methods fit(X, y, init_params=True) Learn model from training data. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values. init_params : bool (default: True) Re-initializes model parameters prior to fitting. Set False to continue training with weights from a previous model fitting. Returns self : object predict(X) Predict targets from X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns target_values : array-like, shape = [n_samples] Predicted target values. predict_proba(X) Predict class probabilities of X from the net input. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns Class probabilties : array-like, shape= [n_samples, n_classes] score(X, y) Compute the prediction accuracy Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values (true class labels). Returns acc : float The prediction accuracy as a float between 0.0 and 1.0 (perfect score). ython","title":"SoftmaxRegression: Multiclass version of logistic regression"},{"location":"user_guide/classifier/SoftmaxRegression/#softmaxregression-multiclass-version-of-logistic-regression","text":"A logistic regression class for multi-class classification tasks. from mlxtend.classifier import SoftmaxRegression","title":"SoftmaxRegression: Multiclass version of logistic regression"},{"location":"user_guide/classifier/SoftmaxRegression/#overview","text":"Softmax Regression (synonyms: Multinomial Logistic , Maximum Entropy Classifier , or just Multi-class Logistic Regression ) is a generalization of logistic regression that we can use for multi-class classification (under the assumption that the classes are mutually exclusive). In contrast, we use the (standard) Logistic Regression model in binary classification tasks. Below is a schematic of a Logistic Regression model, for more details, please see the LogisticRegression manual . In Softmax Regression (SMR), we replace the sigmoid logistic function by the so-called softmax function \\phi_{softmax}(\\cdot) . P(y=j \\mid z^{(i)}) = \\phi_{softmax}(z^{(i)}) = \\frac{e^{z^{(i)}}}{\\sum_{j=0}^{k} e^{z_{k}^{(i)}}}, where we define the net input z as z = w_1x_1 + ... + w_mx_m + b= \\sum_{l=1}^{m} w_l x_l + b= \\mathbf{w}^T\\mathbf{x} + b. ( w is the weight vector, \\mathbf{x} is the feature vector of 1 training sample, and b is the bias unit.) Now, this softmax function computes the probability that this training sample \\mathbf{x}^{(i)} belongs to class j given the weight and net input z^{(i)} . So, we compute the probability p(y = j \\mid \\mathbf{x^{(i)}; w}_j) for each class label in j = 1, \\ldots, k. . Note the normalization term in the denominator which causes these class probabilities to sum up to one. To illustrate the concept of softmax, let us walk through a concrete example. Let's assume we have a training set consisting of 4 samples from 3 different classes (0, 1, and 2) x_0 \\rightarrow \\text{class }0 x_1 \\rightarrow \\text{class }1 x_2 \\rightarrow \\text{class }2 x_3 \\rightarrow \\text{class }2 import numpy as np y = np.array([0, 1, 2, 2]) First, we want to encode the class labels into a format that we can more easily work with; we apply one-hot encoding: y_enc = (np.arange(np.max(y) + 1) == y[:, None]).astype(float) print('one-hot encoding:\\n', y_enc) one-hot encoding: [[ 1. 0. 0.] [ 0. 1. 0.] [ 0. 0. 1.] [ 0. 0. 1.]] A sample that belongs to class 0 (the first row) has a 1 in the first cell, a sample that belongs to class 2 has a 1 in the second cell of its row, and so forth. Next, let us define the feature matrix of our 4 training samples. Here, we assume that our dataset consists of 2 features; thus, we create a 4x2 dimensional matrix of our samples and features. Similarly, we create a 2x3 dimensional weight matrix (one row per feature and one column for each class). X = np.array([[0.1, 0.5], [1.1, 2.3], [-1.1, -2.3], [-1.5, -2.5]]) W = np.array([[0.1, 0.2, 0.3], [0.1, 0.2, 0.3]]) bias = np.array([0.01, 0.1, 0.1]) print('Inputs X:\\n', X) print('\\nWeights W:\\n', W) print('\\nbias:\\n', bias) Inputs X: [[ 0.1 0.5] [ 1.1 2.3] [-1.1 -2.3] [-1.5 -2.5]] Weights W: [[ 0.1 0.2 0.3] [ 0.1 0.2 0.3]] bias: [ 0.01 0.1 0.1 ] To compute the net input, we multiply the 4x2 matrix feature matrix X with the 2x3 (n_features x n_classes) weight matrix W , which yields a 4x3 output matrix (n_samples x n_classes) to which we then add the bias unit: \\mathbf{Z} = \\mathbf{X}\\mathbf{W} + \\mathbf{b}. X = np.array([[0.1, 0.5], [1.1, 2.3], [-1.1, -2.3], [-1.5, -2.5]]) W = np.array([[0.1, 0.2, 0.3], [0.1, 0.2, 0.3]]) bias = np.array([0.01, 0.1, 0.1]) print('Inputs X:\\n', X) print('\\nWeights W:\\n', W) print('\\nbias:\\n', bias) Inputs X: [[ 0.1 0.5] [ 1.1 2.3] [-1.1 -2.3] [-1.5 -2.5]] Weights W: [[ 0.1 0.2 0.3] [ 0.1 0.2 0.3]] bias: [ 0.01 0.1 0.1 ] def net_input(X, W, b): return (X.dot(W) + b) net_in = net_input(X, W, bias) print('net input:\\n', net_in) net input: [[ 0.07 0.22 0.28] [ 0.35 0.78 1.12] [-0.33 -0.58 -0.92] [-0.39 -0.7 -1.1 ]] Now, it's time to compute the softmax activation that we discussed earlier: P(y=j \\mid z^{(i)}) = \\phi_{softmax}(z^{(i)}) = \\frac{e^{z^{(i)}}}{\\sum_{j=0}^{k} e^{z_{k}^{(i)}}}. def softmax(z): return (np.exp(z.T) / np.sum(np.exp(z), axis=1)).T smax = softmax(net_in) print('softmax:\\n', smax) softmax: [[ 0.29450637 0.34216758 0.36332605] [ 0.21290077 0.32728332 0.45981591] [ 0.42860913 0.33380113 0.23758974] [ 0.44941979 0.32962558 0.22095463]] As we can see, the values for each sample (row) nicely sum up to 1 now. E.g., we can say that the first sample [ 0.29450637 0.34216758 0.36332605] has a 29.45% probability to belong to class 0. Now, in order to turn these probabilities back into class labels, we could simply take the argmax-index position of each row: [[ 0.29450637 0.34216758 0.36332605 ] -> 2 [ 0.21290077 0.32728332 0.45981591 ] -> 2 [ 0.42860913 0.33380113 0.23758974] -> 0 [ 0.44941979 0.32962558 0.22095463]] -> 0 def to_classlabel(z): return z.argmax(axis=1) print('predicted class labels: ', to_classlabel(smax)) predicted class labels: [2 2 0 0] As we can see, our predictions are terribly wrong, since the correct class labels are [0, 1, 2, 2] . Now, in order to train our logistic model (e.g., via an optimization algorithm such as gradient descent), we need to define a cost function J(\\cdot) that we want to minimize: J(\\mathbf{W}; \\mathbf{b}) = \\frac{1}{n} \\sum_{i=1}^{n} H(T_i, O_i), which is the average of all cross-entropies over our n training samples. The cross-entropy function is defined as H(T_i, O_i) = -\\sum_m T_i \\cdot log(O_i). Here the T stands for \"target\" (i.e., the true class labels) and the O stands for output -- the computed probability via softmax; not the predicted class label. def cross_entropy(output, y_target): return - np.sum(np.log(output) * (y_target), axis=1) xent = cross_entropy(smax, y_enc) print('Cross Entropy:', xent) Cross Entropy: [ 1.22245465 1.11692907 1.43720989 1.50979788] def cost(output, y_target): return np.mean(cross_entropy(output, y_target)) J_cost = cost(smax, y_enc) print('Cost: ', J_cost) Cost: 1.32159787159 In order to learn our softmax model -- determining the weight coefficients -- via gradient descent, we then need to compute the derivative \\nabla \\mathbf{w}_j \\, J(\\mathbf{W}; \\mathbf{b}). I don't want to walk through the tedious details here, but this cost derivative turns out to be simply: \\nabla \\mathbf{w}_j \\, J(\\mathbf{W}; \\mathbf{b}) = \\frac{1}{n} \\sum^{n}_{i=0} \\big[\\mathbf{x}^{(i)}\\ \\big(O_i - T_i \\big) \\big] We can then use the cost derivate to update the weights in opposite direction of the cost gradient with learning rate \\eta : \\mathbf{w}_j := \\mathbf{w}_j - \\eta \\nabla \\mathbf{w}_j \\, J(\\mathbf{W}; \\mathbf{b}) for each class j \\in \\{0, 1, ..., k\\} (note that \\mathbf{w}_j is the weight vector for the class y=j ), and we update the bias units \\mathbf{b}_j := \\mathbf{b}_j - \\eta \\bigg[ \\frac{1}{n} \\sum^{n}_{i=0} \\big(O_i - T_i \\big) \\bigg]. As a penalty against complexity, an approach to reduce the variance of our model and decrease the degree of overfitting by adding additional bias, we can further add a regularization term such as the L2 term with the regularization parameter \\lambda : L2: \\frac{\\lambda}{2} ||\\mathbf{w}||_{2}^{2} , where ||\\mathbf{w}||_{2}^{2} = \\sum^{m}_{l=0} \\sum^{k}_{j=0} w_{i, j} so that our cost function becomes J(\\mathbf{W}; \\mathbf{b}) = \\frac{1}{n} \\sum_{i=1}^{n} H(T_i, O_i) + \\frac{\\lambda}{2} ||\\mathbf{w}||_{2}^{2} and we define the \"regularized\" weight update as \\mathbf{w}_j := \\mathbf{w}_j - \\eta \\big[\\nabla \\mathbf{w}_j \\, J(\\mathbf{W}) + \\lambda \\mathbf{w}_j \\big]. (Please note that we don't regularize the bias term.)","title":"Overview"},{"location":"user_guide/classifier/SoftmaxRegression/#example-1-gradient-descent","text":"from mlxtend.data import iris_data from mlxtend.plotting import plot_decision_regions from mlxtend.classifier import SoftmaxRegression import matplotlib.pyplot as plt # Loading Data X, y = iris_data() X = X[:, [0, 3]] # sepal length and petal width # standardize X[:,0] = (X[:,0] - X[:,0].mean()) / X[:,0].std() X[:,1] = (X[:,1] - X[:,1].mean()) / X[:,1].std() lr = SoftmaxRegression(eta=0.01, epochs=500, minibatches=1, random_seed=1, print_progress=3) lr.fit(X, y) plot_decision_regions(X, y, clf=lr) plt.title('Softmax Regression - Gradient Descent') plt.show() plt.plot(range(len(lr.cost_)), lr.cost_) plt.xlabel('Iterations') plt.ylabel('Cost') plt.show() Iteration: 500/500 | Cost 0.06 | Elapsed: 0:00:00 | ETA: 0:00:00","title":"Example 1 - Gradient Descent"},{"location":"user_guide/classifier/SoftmaxRegression/#predicting-class-labels","text":"y_pred = lr.predict(X) print('Last 3 Class Labels: %s' % y_pred[-3:]) Last 3 Class Labels: [2 2 2]","title":"Predicting Class Labels"},{"location":"user_guide/classifier/SoftmaxRegression/#predicting-class-probabilities","text":"y_pred = lr.predict_proba(X) print('Last 3 Class Labels:\\n %s' % y_pred[-3:]) Last 3 Class Labels: [[ 9.18728149e-09 1.68894679e-02 9.83110523e-01] [ 2.97052325e-11 7.26356627e-04 9.99273643e-01] [ 1.57464093e-06 1.57779528e-01 8.42218897e-01]]","title":"Predicting Class Probabilities"},{"location":"user_guide/classifier/SoftmaxRegression/#example-2-stochastic-gradient-descent","text":"from mlxtend.data import iris_data from mlxtend.plotting import plot_decision_regions from mlxtend.classifier import SoftmaxRegression import matplotlib.pyplot as plt # Loading Data X, y = iris_data() X = X[:, [0, 3]] # sepal length and petal width # standardize X[:,0] = (X[:,0] - X[:,0].mean()) / X[:,0].std() X[:,1] = (X[:,1] - X[:,1].mean()) / X[:,1].std() lr = SoftmaxRegression(eta=0.01, epochs=300, minibatches=len(y), random_seed=1) lr.fit(X, y) plot_decision_regions(X, y, clf=lr) plt.title('Softmax Regression - Stochastic Gradient Descent') plt.show() plt.plot(range(len(lr.cost_)), lr.cost_) plt.xlabel('Iterations') plt.ylabel('Cost') plt.show()","title":"Example 2 - Stochastic Gradient Descent"},{"location":"user_guide/classifier/SoftmaxRegression/#api","text":"SoftmaxRegression(eta=0.01, epochs=50, l2=0.0, minibatches=1, n_classes=None, random_seed=None, print_progress=0) Softmax regression classifier. Parameters eta : float (default: 0.01) Learning rate (between 0.0 and 1.0) epochs : int (default: 50) Passes over the training dataset. Prior to each epoch, the dataset is shuffled if minibatches > 1 to prevent cycles in stochastic gradient descent. l2 : float Regularization parameter for L2 regularization. No regularization if l2=0.0. minibatches : int (default: 1) The number of minibatches for gradient-based optimization. If 1: Gradient Descent learning If len(y): Stochastic Gradient Descent (SGD) online learning If 1 < minibatches < len(y): SGD Minibatch learning n_classes : int (default: None) A positive integer to declare the number of class labels if not all class labels are present in a partial training set. Gets the number of class labels automatically if None. random_seed : int (default: None) Set random state for shuffling and initializing the weights. print_progress : int (default: 0) Prints progress in fitting to stderr. 0: No output 1: Epochs elapsed and cost 2: 1 plus time elapsed 3: 2 plus estimated time until completion Attributes w_ : 2d-array, shape={n_features, 1} Model weights after fitting. b_ : 1d-array, shape={1,} Bias unit after fitting. cost_ : list List of floats, the average cross_entropy for each epoch. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/classifier/SoftmaxRegression/","title":"API"},{"location":"user_guide/classifier/SoftmaxRegression/#methods","text":"fit(X, y, init_params=True) Learn model from training data. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values. init_params : bool (default: True) Re-initializes model parameters prior to fitting. Set False to continue training with weights from a previous model fitting. Returns self : object predict(X) Predict targets from X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns target_values : array-like, shape = [n_samples] Predicted target values. predict_proba(X) Predict class probabilities of X from the net input. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns Class probabilties : array-like, shape= [n_samples, n_classes] score(X, y) Compute the prediction accuracy Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values (true class labels). Returns acc : float The prediction accuracy as a float between 0.0 and 1.0 (perfect score). ython","title":"Methods"},{"location":"user_guide/classifier/StackingCVClassifier/","text":"StackingCVClassifier: Stacking with cross-validation An ensemble-learning meta-classifier for stacking using cross-validation to prepare the inputs for the level-2 classifier to prevent overfitting. from mlxtend.classifier import StackingCVClassifier Overview Stacking is an ensemble learning technique to combine multiple classification models via a meta-classifier. The StackingCVClassifier extends the standard stacking algorithm (implemented as StackingClassifier ) using cross-validation to prepare the input data for the level-2 classifier. In the standard stacking procedure, the first-level classifiers are fit to the same training set that is used prepare the inputs for the second-level classifier, which may lead to overfitting. The StackingCVClassifier , however, uses the concept of cross-validation: the dataset is split into k folds, and in k successive rounds, k-1 folds are used to fit the first level classifier; in each round, the first-level classifiers are then applied to the remaining 1 subset that was not used for model fitting in each iteration. The resulting predictions are then stacked and provided -- as input data -- to the second-level classifier. After the training of the StackingCVClassifier , the first-level classifiers are fit to the entire dataset as illustrated in the figure below. More formally, the Stacking Cross-Validation algorithm can be summarized as follows (source: [1]): References [1] Tang, J., S. Alelyani, and H. Liu. \" Data Classification: Algorithms and Applications. \" Data Mining and Knowledge Discovery Series, CRC Press (2015): pp. 498-500. [2] Wolpert, David H. \" Stacked generalization. \" Neural networks 5.2 (1992): 241-259. [3] Marios Michailidis (2017), StackNet, StackNet Meta Modelling Framework, https://github.com/kaz-Anova/StackNet Example 1 - Simple Stacking CV Classification from sklearn import datasets iris = datasets.load_iris() X, y = iris.data[:, 1:3], iris.target from sklearn import model_selection from sklearn.linear_model import LogisticRegression from sklearn.neighbors import KNeighborsClassifier from sklearn.naive_bayes import GaussianNB from sklearn.ensemble import RandomForestClassifier from mlxtend.classifier import StackingCVClassifier import numpy as np import warnings warnings.simplefilter('ignore') RANDOM_SEED = 42 clf1 = KNeighborsClassifier(n_neighbors=1) clf2 = RandomForestClassifier(random_state=RANDOM_SEED) clf3 = GaussianNB() lr = LogisticRegression() # Starting from v0.16.0, StackingCVRegressor supports # `random_state` to get deterministic result. sclf = StackingCVClassifier(classifiers=[clf1, clf2, clf3], meta_classifier=lr, random_state=RANDOM_SEED) print('3-fold cross validation:\\n') for clf, label in zip([clf1, clf2, clf3, sclf], ['KNN', 'Random Forest', 'Naive Bayes', 'StackingClassifier']): scores = model_selection.cross_val_score(clf, X, y, cv=3, scoring='accuracy') print(\"Accuracy: %0.2f (+/- %0.2f) [%s]\" % (scores.mean(), scores.std(), label)) 3-fold cross validation: Accuracy: 0.91 (+/- 0.01) [KNN] Accuracy: 0.95 (+/- 0.01) [Random Forest] Accuracy: 0.91 (+/- 0.02) [Naive Bayes] Accuracy: 0.93 (+/- 0.02) [StackingClassifier] import matplotlib.pyplot as plt from mlxtend.plotting import plot_decision_regions import matplotlib.gridspec as gridspec import itertools gs = gridspec.GridSpec(2, 2) fig = plt.figure(figsize=(10,8)) for clf, lab, grd in zip([clf1, clf2, clf3, sclf], ['KNN', 'Random Forest', 'Naive Bayes', 'StackingCVClassifier'], itertools.product([0, 1], repeat=2)): clf.fit(X, y) ax = plt.subplot(gs[grd[0], grd[1]]) fig = plot_decision_regions(X=X, y=y, clf=clf) plt.title(lab) plt.show() Example 2 - Using Probabilities as Meta-Features Alternatively, the class-probabilities of the first-level classifiers can be used to train the meta-classifier (2nd-level classifier) by setting use_probas=True . For example, in a 3-class setting with 2 level-1 classifiers, these classifiers may make the following \"probability\" predictions for 1 training sample: classifier 1: [0.2, 0.5, 0.3] classifier 2: [0.3, 0.4, 0.4] This results in k features, where k = [n_classes * n_classifiers], by stacking these level-1 probabilities: [0.2, 0.5, 0.3, 0.3, 0.4, 0.4] clf1 = KNeighborsClassifier(n_neighbors=1) clf2 = RandomForestClassifier(random_state=1) clf3 = GaussianNB() lr = LogisticRegression() sclf = StackingCVClassifier(classifiers=[clf1, clf2, clf3], use_probas=True, meta_classifier=lr, random_state=42) print('3-fold cross validation:\\n') for clf, label in zip([clf1, clf2, clf3, sclf], ['KNN', 'Random Forest', 'Naive Bayes', 'StackingClassifier']): scores = model_selection.cross_val_score(clf, X, y, cv=3, scoring='accuracy') print(\"Accuracy: %0.2f (+/- %0.2f) [%s]\" % (scores.mean(), scores.std(), label)) 3-fold cross validation: Accuracy: 0.91 (+/- 0.01) [KNN] Accuracy: 0.95 (+/- 0.01) [Random Forest] Accuracy: 0.91 (+/- 0.02) [Naive Bayes] Accuracy: 0.95 (+/- 0.02) [StackingClassifier] Example 3 - Stacked CV Classification and GridSearch The stack allows tuning hyper parameters of the base and meta models! A full list of tunable parameters can be obtained via estimator.get_params().keys() . from sklearn.linear_model import LogisticRegression from sklearn.neighbors import KNeighborsClassifier from sklearn.naive_bayes import GaussianNB from sklearn.ensemble import RandomForestClassifier from sklearn.model_selection import GridSearchCV from mlxtend.classifier import StackingCVClassifier # Initializing models clf1 = KNeighborsClassifier(n_neighbors=1) clf2 = RandomForestClassifier(random_state=RANDOM_SEED) clf3 = GaussianNB() lr = LogisticRegression() sclf = StackingCVClassifier(classifiers=[clf1, clf2, clf3], meta_classifier=lr, random_state=42) params = {'kneighborsclassifier__n_neighbors': [1, 5], 'randomforestclassifier__n_estimators': [10, 50], 'meta_classifier__C': [0.1, 10.0]} grid = GridSearchCV(estimator=sclf, param_grid=params, cv=5, refit=True) grid.fit(X, y) cv_keys = ('mean_test_score', 'std_test_score', 'params') for r, _ in enumerate(grid.cv_results_['mean_test_score']): print(\"%0.3f +/- %0.2f %r\" % (grid.cv_results_[cv_keys[0]][r], grid.cv_results_[cv_keys[1]][r] / 2.0, grid.cv_results_[cv_keys[2]][r])) print('Best parameters: %s' % grid.best_params_) print('Accuracy: %.2f' % grid.best_score_) 0.947 +/- 0.03 {'kneighborsclassifier__n_neighbors': 1, 'meta_classifier__C': 0.1, 'randomforestclassifier__n_estimators': 10} 0.933 +/- 0.02 {'kneighborsclassifier__n_neighbors': 1, 'meta_classifier__C': 0.1, 'randomforestclassifier__n_estimators': 50} 0.940 +/- 0.02 {'kneighborsclassifier__n_neighbors': 1, 'meta_classifier__C': 10.0, 'randomforestclassifier__n_estimators': 10} 0.940 +/- 0.02 {'kneighborsclassifier__n_neighbors': 1, 'meta_classifier__C': 10.0, 'randomforestclassifier__n_estimators': 50} 0.953 +/- 0.02 {'kneighborsclassifier__n_neighbors': 5, 'meta_classifier__C': 0.1, 'randomforestclassifier__n_estimators': 10} 0.953 +/- 0.02 {'kneighborsclassifier__n_neighbors': 5, 'meta_classifier__C': 0.1, 'randomforestclassifier__n_estimators': 50} 0.953 +/- 0.02 {'kneighborsclassifier__n_neighbors': 5, 'meta_classifier__C': 10.0, 'randomforestclassifier__n_estimators': 10} 0.953 +/- 0.02 {'kneighborsclassifier__n_neighbors': 5, 'meta_classifier__C': 10.0, 'randomforestclassifier__n_estimators': 50} Best parameters: {'kneighborsclassifier__n_neighbors': 5, 'meta_classifier__C': 0.1, 'randomforestclassifier__n_estimators': 10} Accuracy: 0.95 In case we are planning to use a regression algorithm multiple times, all we need to do is to add an additional number suffix in the parameter grid as shown below: from sklearn.model_selection import GridSearchCV # Initializing models clf1 = KNeighborsClassifier(n_neighbors=1) clf2 = RandomForestClassifier(random_state=RANDOM_SEED) clf3 = GaussianNB() lr = LogisticRegression() sclf = StackingCVClassifier(classifiers=[clf1, clf1, clf2, clf3], meta_classifier=lr, random_state=RANDOM_SEED) params = {'kneighborsclassifier-1__n_neighbors': [1, 5], 'kneighborsclassifier-2__n_neighbors': [1, 5], 'randomforestclassifier__n_estimators': [10, 50], 'meta_classifier__C': [0.1, 10.0]} grid = GridSearchCV(estimator=sclf, param_grid=params, cv=5, refit=True) grid.fit(X, y) cv_keys = ('mean_test_score', 'std_test_score', 'params') for r, _ in enumerate(grid.cv_results_['mean_test_score']): print(\"%0.3f +/- %0.2f %r\" % (grid.cv_results_[cv_keys[0]][r], grid.cv_results_[cv_keys[1]][r] / 2.0, grid.cv_results_[cv_keys[2]][r])) print('Best parameters: %s' % grid.best_params_) print('Accuracy: %.2f' % grid.best_score_) 0.940 +/- 0.02 {'kneighborsclassifier-1__n_neighbors': 1, 'kneighborsclassifier-2__n_neighbors': 1, 'meta_classifier__C': 0.1, 'randomforestclassifier__n_estimators': 10} 0.940 +/- 0.02 {'kneighborsclassifier-1__n_neighbors': 1, 'kneighborsclassifier-2__n_neighbors': 1, 'meta_classifier__C': 0.1, 'randomforestclassifier__n_estimators': 50} 0.940 +/- 0.02 {'kneighborsclassifier-1__n_neighbors': 1, 'kneighborsclassifier-2__n_neighbors': 1, 'meta_classifier__C': 10.0, 'randomforestclassifier__n_estimators': 10} 0.940 +/- 0.02 {'kneighborsclassifier-1__n_neighbors': 1, 'kneighborsclassifier-2__n_neighbors': 1, 'meta_classifier__C': 10.0, 'randomforestclassifier__n_estimators': 50} 0.960 +/- 0.02 {'kneighborsclassifier-1__n_neighbors': 1, 'kneighborsclassifier-2__n_neighbors': 5, 'meta_classifier__C': 0.1, 'randomforestclassifier__n_estimators': 10} 0.953 +/- 0.02 {'kneighborsclassifier-1__n_neighbors': 1, 'kneighborsclassifier-2__n_neighbors': 5, 'meta_classifier__C': 0.1, 'randomforestclassifier__n_estimators': 50} 0.953 +/- 0.02 {'kneighborsclassifier-1__n_neighbors': 1, 'kneighborsclassifier-2__n_neighbors': 5, 'meta_classifier__C': 10.0, 'randomforestclassifier__n_estimators': 10} 0.953 +/- 0.02 {'kneighborsclassifier-1__n_neighbors': 1, 'kneighborsclassifier-2__n_neighbors': 5, 'meta_classifier__C': 10.0, 'randomforestclassifier__n_estimators': 50} 0.960 +/- 0.02 {'kneighborsclassifier-1__n_neighbors': 5, 'kneighborsclassifier-2__n_neighbors': 1, 'meta_classifier__C': 0.1, 'randomforestclassifier__n_estimators': 10} 0.953 +/- 0.02 {'kneighborsclassifier-1__n_neighbors': 5, 'kneighborsclassifier-2__n_neighbors': 1, 'meta_classifier__C': 0.1, 'randomforestclassifier__n_estimators': 50} 0.953 +/- 0.02 {'kneighborsclassifier-1__n_neighbors': 5, 'kneighborsclassifier-2__n_neighbors': 1, 'meta_classifier__C': 10.0, 'randomforestclassifier__n_estimators': 10} 0.953 +/- 0.02 {'kneighborsclassifier-1__n_neighbors': 5, 'kneighborsclassifier-2__n_neighbors': 1, 'meta_classifier__C': 10.0, 'randomforestclassifier__n_estimators': 50} 0.953 +/- 0.02 {'kneighborsclassifier-1__n_neighbors': 5, 'kneighborsclassifier-2__n_neighbors': 5, 'meta_classifier__C': 0.1, 'randomforestclassifier__n_estimators': 10} 0.953 +/- 0.02 {'kneighborsclassifier-1__n_neighbors': 5, 'kneighborsclassifier-2__n_neighbors': 5, 'meta_classifier__C': 0.1, 'randomforestclassifier__n_estimators': 50} 0.953 +/- 0.02 {'kneighborsclassifier-1__n_neighbors': 5, 'kneighborsclassifier-2__n_neighbors': 5, 'meta_classifier__C': 10.0, 'randomforestclassifier__n_estimators': 10} 0.953 +/- 0.02 {'kneighborsclassifier-1__n_neighbors': 5, 'kneighborsclassifier-2__n_neighbors': 5, 'meta_classifier__C': 10.0, 'randomforestclassifier__n_estimators': 50} Best parameters: {'kneighborsclassifier-1__n_neighbors': 1, 'kneighborsclassifier-2__n_neighbors': 5, 'meta_classifier__C': 0.1, 'randomforestclassifier__n_estimators': 10} Accuracy: 0.96 Note The StackingClassifier also enables grid search over the classifiers argument. When there are level-mixed hyperparameters, GridSearchCV will try to replace hyperparameters in a top-down order, i.e., classifers -> single base classifier -> classifier hyperparameter. For instance, given a hyperparameter grid such as params = {'randomforestclassifier__n_estimators': [1, 100], 'classifiers': [(clf1, clf1, clf1), (clf2, clf3)]} it will first use the instance settings of either (clf1, clf1, clf1) or (clf2, clf3). Then it will replace the 'n_estimators' settings for a matching classifier based on 'randomforestclassifier__n_estimators': [1, 100] . Example 4 - Stacking of Classifiers that Operate on Different Feature Subsets The different level-1 classifiers can be fit to different subsets of features in the training dataset. The following example illustrates how this can be done on a technical level using scikit-learn pipelines and the ColumnSelector : from sklearn.datasets import load_iris from mlxtend.classifier import StackingCVClassifier from mlxtend.feature_selection import ColumnSelector from sklearn.pipeline import make_pipeline from sklearn.linear_model import LogisticRegression iris = load_iris() X = iris.data y = iris.target pipe1 = make_pipeline(ColumnSelector(cols=(0, 2)), LogisticRegression()) pipe2 = make_pipeline(ColumnSelector(cols=(1, 2, 3)), LogisticRegression()) sclf = StackingCVClassifier(classifiers=[pipe1, pipe2], meta_classifier=LogisticRegression(), random_state=42) sclf.fit(X, y) StackingCVClassifier(classifiers=[Pipeline(memory=None, steps=[('columnselector', ColumnSelector(cols=(0, 2), drop_axis=False)), ('logisticregression', LogisticRegression(C=1.0, class_weight=None, dual=False, fit_intercept=True, intercept_scaling=1, l1_ratio=None, max_iter=100, multi_class='auto', n_jobs=None, penalty='l2', random_state=None, solver='lbfgs', tol=0.0... fit_intercept=True, intercept_scaling=1, l1_ratio=None, max_iter=100, multi_class='auto', n_jobs=None, penalty='l2', random_state=None, solver='lbfgs', tol=0.0001, verbose=0, warm_start=False), n_jobs=None, pre_dispatch='2*n_jobs', random_state=42, shuffle=True, store_train_meta_features=False, stratify=True, use_clones=True, use_features_in_secondary=False, use_probas=False, verbose=0) Example 5 -- ROC Curve with decision_function Like other scikit-learn classifiers, the StackingCVClassifier has an decision_function method that can be used for plotting ROC curves. Note that the decision_function expects and requires the meta-classifier to implement a decision_function . from sklearn import model_selection from sklearn.linear_model import LogisticRegression from sklearn.neighbors import KNeighborsClassifier from sklearn.svm import SVC from sklearn.ensemble import RandomForestClassifier from mlxtend.classifier import StackingCVClassifier from sklearn.metrics import roc_curve, auc import numpy as np from sklearn.model_selection import train_test_split from sklearn import datasets from sklearn.preprocessing import label_binarize from sklearn.multiclass import OneVsRestClassifier iris = datasets.load_iris() X, y = iris.data[:, [0, 1]], iris.target # Binarize the output y = label_binarize(y, classes=[0, 1, 2]) n_classes = y.shape[1] RANDOM_SEED = 42 X_train, X_test, y_train, y_test = train_test_split( X, y, test_size=0.33, random_state=RANDOM_SEED) clf1 = LogisticRegression() clf2 = RandomForestClassifier(random_state=RANDOM_SEED) clf3 = SVC(random_state=RANDOM_SEED) lr = LogisticRegression() sclf = StackingCVClassifier(classifiers=[clf1, clf2, clf3], meta_classifier=lr) # Learn to predict each class against the other classifier = OneVsRestClassifier(sclf) Using predict_proba() y_score = classifier.fit(X_train, y_train).predict_proba(X_test) # Compute ROC curve and ROC area for each class fpr = dict() tpr = dict() roc_auc = dict() for i in range(n_classes): fpr[i], tpr[i], _ = roc_curve(y_test[:, i], y_score[:, i]) roc_auc[i] = auc(fpr[i], tpr[i]) # Compute micro-average ROC curve and ROC area fpr[\"micro\"], tpr[\"micro\"], _ = roc_curve(y_test.ravel(), y_score.ravel()) roc_auc[\"micro\"] = auc(fpr[\"micro\"], tpr[\"micro\"]) plt.figure() lw = 2 plt.plot(fpr[2], tpr[2], color='darkorange', lw=lw, label='ROC curve (area = %0.2f)' % roc_auc[2]) plt.plot([0, 1], [0, 1], color='navy', lw=lw, linestyle='--') plt.xlim([0.0, 1.0]) plt.ylim([0.0, 1.05]) plt.xlabel('False Positive Rate') plt.ylabel('True Positive Rate') plt.title('Receiver operating characteristic example') plt.legend(loc=\"lower right\") plt.show() Using decision_function() y_score = classifier.fit(X_train, y_train).decision_function(X_test) # Compute ROC curve and ROC area for each class fpr = dict() tpr = dict() roc_auc = dict() for i in range(n_classes): fpr[i], tpr[i], _ = roc_curve(y_test[:, i], y_score[:, i]) roc_auc[i] = auc(fpr[i], tpr[i]) # Compute micro-average ROC curve and ROC area fpr[\"micro\"], tpr[\"micro\"], _ = roc_curve(y_test.ravel(), y_score.ravel()) roc_auc[\"micro\"] = auc(fpr[\"micro\"], tpr[\"micro\"]) plt.figure() lw = 2 plt.plot(fpr[2], tpr[2], color='darkorange', lw=lw, label='ROC curve (area = %0.2f)' % roc_auc[2]) plt.plot([0, 1], [0, 1], color='navy', lw=lw, linestyle='--') plt.xlim([0.0, 1.0]) plt.ylim([0.0, 1.05]) plt.xlabel('False Positive Rate') plt.ylabel('True Positive Rate') plt.title('Receiver operating characteristic example') plt.legend(loc=\"lower right\") plt.show() API StackingCVClassifier(classifiers, meta_classifier, use_probas=False, drop_proba_col=None, cv=2, shuffle=True, random_state=None, stratify=True, verbose=0, use_features_in_secondary=False, store_train_meta_features=False, use_clones=True, n_jobs=None, pre_dispatch='2 n_jobs')* A 'Stacking Cross-Validation' classifier for scikit-learn estimators. New in mlxtend v0.4.3 Parameters classifiers : array-like, shape = [n_classifiers] A list of classifiers. Invoking the fit method on the StackingCVClassifer will fit clones of these original classifiers that will be stored in the class attribute self.clfs_ . meta_classifier : object The meta-classifier to be fitted on the ensemble of classifiers use_probas : bool (default: False) If True, trains meta-classifier based on predicted probabilities instead of class labels. drop_proba_col : string (default: None) Drops extra \"probability\" column in the feature set, because it is redundant: p(y_c) = 1 - p(y_1) + p(y_2) + ... + p(y_{c-1}). This can be useful for meta-classifiers that are sensitive to perfectly collinear features. If last , drops last probability column. If first , drops first probability column. Only relevant if use_probas=True . cv : int, cross-validation generator or an iterable, optional (default: 2) Determines the cross-validation splitting strategy. Possible inputs for cv are: - None, to use the default 2-fold cross validation, - integer, to specify the number of folds in a (Stratified)KFold , - An object to be used as a cross-validation generator. - An iterable yielding train, test splits. For integer/None inputs, it will use either a KFold or StratifiedKFold cross validation depending the value of stratify argument. shuffle : bool (default: True) If True, and the cv argument is integer, the training data will be shuffled at fitting stage prior to cross-validation. If the cv argument is a specific cross validation technique, this argument is omitted. random_state : int, RandomState instance or None, optional (default: None) Constrols the randomness of the cv splitter. Used when cv is integer and shuffle=True . New in v0.16.0. stratify : bool (default: True) If True, and the cv argument is integer it will follow a stratified K-Fold cross validation technique. If the cv argument is a specific cross validation technique, this argument is omitted. verbose : int, optional (default=0) Controls the verbosity of the building process. - verbose=0 (default): Prints nothing - verbose=1 : Prints the number & name of the regressor being fitted and which fold is currently being used for fitting - verbose=2 : Prints info about the parameters of the regressor being fitted - verbose>2 : Changes verbose param of the underlying regressor to self.verbose - 2 use_features_in_secondary : bool (default: False) If True, the meta-classifier will be trained both on the predictions of the original classifiers and the original dataset. If False, the meta-classifier will be trained only on the predictions of the original classifiers. store_train_meta_features : bool (default: False) If True, the meta-features computed from the training data used for fitting the meta-classifier stored in the self.train_meta_features_ array, which can be accessed after calling fit . use_clones : bool (default: True) Clones the classifiers for stacking classification if True (default) or else uses the original ones, which will be refitted on the dataset upon calling the fit method. Hence, if use_clones=True, the original input classifiers will remain unmodified upon using the StackingCVClassifier's fit method. Setting use_clones=False is recommended if you are working with estimators that are supporting the scikit-learn fit/predict API interface but are not compatible to scikit-learn's clone function. n_jobs : int or None, optional (default=None) The number of CPUs to use to do the computation. None means 1 unless in a :obj: joblib.parallel_backend context. -1 means using all processors. See :term: Glossary <n_jobs> for more details. New in v0.16.0. pre_dispatch : int, or string, optional Controls the number of jobs that get dispatched during parallel execution. Reducing this number can be useful to avoid an explosion of memory consumption when more jobs get dispatched than CPUs can process. This parameter can be: - None, in which case all the jobs are immediately created and spawned. Use this for lightweight and fast-running jobs, to avoid delays due to on-demand spawning of the jobs - An int, giving the exact number of total jobs that are spawned - A string, giving an expression as a function of n_jobs, as in '2*n_jobs' New in v0.16.0. Attributes clfs_ : list, shape=[n_classifiers] Fitted classifiers (clones of the original classifiers) meta_clf_ : estimator Fitted meta-classifier (clone of the original meta-estimator) train_meta_features : numpy array, shape = [n_samples, n_classifiers] meta-features for training data, where n_samples is the number of samples in training data and n_classifiers is the number of classfiers. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/classifier/StackingCVClassifier/ Methods decision_function(X) Predict class confidence scores for X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns scores : shape=(n_samples,) if n_classes == 2 else (n_samples, n_classes). Confidence scores per (sample, class) combination. In the binary case, confidence score for self.classes_[1] where >0 means this class would be predicted. fit(X, y, groups=None, sample_weight=None) Fit ensemble classifers and the meta-classifier. Parameters X : numpy array, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : numpy array, shape = [n_samples] Target values. groups : numpy array/None, shape = [n_samples] The group that each sample belongs to. This is used by specific folding strategies such as GroupKFold() sample_weight : array-like, shape = [n_samples], optional Sample weights passed as sample_weights to each regressor in the regressors list as well as the meta_regressor. Raises error if some regressor does not support sample_weight in the fit() method. Returns self : object fit_transform(X, y=None, fit_params) Fit to data, then transform it. Fits transformer to X and y with optional parameters fit_params and returns a transformed version of X. Parameters X : numpy array of shape [n_samples, n_features] Training set. y : numpy array of shape [n_samples] Target values. **fit_params : dict Additional fit parameters. Returns X_new : numpy array of shape [n_samples, n_features_new] Transformed array. get_params(deep=True) Return estimator parameter names for GridSearch support. predict(X) Predict target values for X. Parameters X : numpy array, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns labels : array-like, shape = [n_samples] Predicted class labels. predict_meta_features(X) Get meta-features of test-data. Parameters X : numpy array, shape = [n_samples, n_features] Test vectors, where n_samples is the number of samples and n_features is the number of features. Returns meta-features : numpy array, shape = [n_samples, n_classifiers] Returns the meta-features for test data. predict_proba(X) Predict class probabilities for X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns proba : array-like, shape = [n_samples, n_classes] or a list of n_outputs of such arrays if n_outputs > 1. Probability for each class per sample. score(X, y, sample_weight=None) Return the mean accuracy on the given test data and labels. In multi-label classification, this is the subset accuracy which is a harsh metric since you require for each sample that each label set be correctly predicted. Parameters X : array-like of shape (n_samples, n_features) Test samples. y : array-like of shape (n_samples,) or (n_samples, n_outputs) True labels for X. sample_weight : array-like of shape (n_samples,), default=None Sample weights. Returns score : float Mean accuracy of self.predict(X) wrt. y. set_params( params) Set the parameters of this estimator. Valid parameter keys can be listed with get_params() . Returns self Properties named_classifiers None","title":"StackingCVClassifier: Stacking with cross-validation"},{"location":"user_guide/classifier/StackingCVClassifier/#stackingcvclassifier-stacking-with-cross-validation","text":"An ensemble-learning meta-classifier for stacking using cross-validation to prepare the inputs for the level-2 classifier to prevent overfitting. from mlxtend.classifier import StackingCVClassifier","title":"StackingCVClassifier: Stacking with cross-validation"},{"location":"user_guide/classifier/StackingCVClassifier/#overview","text":"Stacking is an ensemble learning technique to combine multiple classification models via a meta-classifier. The StackingCVClassifier extends the standard stacking algorithm (implemented as StackingClassifier ) using cross-validation to prepare the input data for the level-2 classifier. In the standard stacking procedure, the first-level classifiers are fit to the same training set that is used prepare the inputs for the second-level classifier, which may lead to overfitting. The StackingCVClassifier , however, uses the concept of cross-validation: the dataset is split into k folds, and in k successive rounds, k-1 folds are used to fit the first level classifier; in each round, the first-level classifiers are then applied to the remaining 1 subset that was not used for model fitting in each iteration. The resulting predictions are then stacked and provided -- as input data -- to the second-level classifier. After the training of the StackingCVClassifier , the first-level classifiers are fit to the entire dataset as illustrated in the figure below. More formally, the Stacking Cross-Validation algorithm can be summarized as follows (source: [1]):","title":"Overview"},{"location":"user_guide/classifier/StackingCVClassifier/#references","text":"[1] Tang, J., S. Alelyani, and H. Liu. \" Data Classification: Algorithms and Applications. \" Data Mining and Knowledge Discovery Series, CRC Press (2015): pp. 498-500. [2] Wolpert, David H. \" Stacked generalization. \" Neural networks 5.2 (1992): 241-259. [3] Marios Michailidis (2017), StackNet, StackNet Meta Modelling Framework, https://github.com/kaz-Anova/StackNet","title":"References"},{"location":"user_guide/classifier/StackingCVClassifier/#example-1-simple-stacking-cv-classification","text":"from sklearn import datasets iris = datasets.load_iris() X, y = iris.data[:, 1:3], iris.target from sklearn import model_selection from sklearn.linear_model import LogisticRegression from sklearn.neighbors import KNeighborsClassifier from sklearn.naive_bayes import GaussianNB from sklearn.ensemble import RandomForestClassifier from mlxtend.classifier import StackingCVClassifier import numpy as np import warnings warnings.simplefilter('ignore') RANDOM_SEED = 42 clf1 = KNeighborsClassifier(n_neighbors=1) clf2 = RandomForestClassifier(random_state=RANDOM_SEED) clf3 = GaussianNB() lr = LogisticRegression() # Starting from v0.16.0, StackingCVRegressor supports # `random_state` to get deterministic result. sclf = StackingCVClassifier(classifiers=[clf1, clf2, clf3], meta_classifier=lr, random_state=RANDOM_SEED) print('3-fold cross validation:\\n') for clf, label in zip([clf1, clf2, clf3, sclf], ['KNN', 'Random Forest', 'Naive Bayes', 'StackingClassifier']): scores = model_selection.cross_val_score(clf, X, y, cv=3, scoring='accuracy') print(\"Accuracy: %0.2f (+/- %0.2f) [%s]\" % (scores.mean(), scores.std(), label)) 3-fold cross validation: Accuracy: 0.91 (+/- 0.01) [KNN] Accuracy: 0.95 (+/- 0.01) [Random Forest] Accuracy: 0.91 (+/- 0.02) [Naive Bayes] Accuracy: 0.93 (+/- 0.02) [StackingClassifier] import matplotlib.pyplot as plt from mlxtend.plotting import plot_decision_regions import matplotlib.gridspec as gridspec import itertools gs = gridspec.GridSpec(2, 2) fig = plt.figure(figsize=(10,8)) for clf, lab, grd in zip([clf1, clf2, clf3, sclf], ['KNN', 'Random Forest', 'Naive Bayes', 'StackingCVClassifier'], itertools.product([0, 1], repeat=2)): clf.fit(X, y) ax = plt.subplot(gs[grd[0], grd[1]]) fig = plot_decision_regions(X=X, y=y, clf=clf) plt.title(lab) plt.show()","title":"Example 1 - Simple Stacking CV Classification"},{"location":"user_guide/classifier/StackingCVClassifier/#example-2-using-probabilities-as-meta-features","text":"Alternatively, the class-probabilities of the first-level classifiers can be used to train the meta-classifier (2nd-level classifier) by setting use_probas=True . For example, in a 3-class setting with 2 level-1 classifiers, these classifiers may make the following \"probability\" predictions for 1 training sample: classifier 1: [0.2, 0.5, 0.3] classifier 2: [0.3, 0.4, 0.4] This results in k features, where k = [n_classes * n_classifiers], by stacking these level-1 probabilities: [0.2, 0.5, 0.3, 0.3, 0.4, 0.4] clf1 = KNeighborsClassifier(n_neighbors=1) clf2 = RandomForestClassifier(random_state=1) clf3 = GaussianNB() lr = LogisticRegression() sclf = StackingCVClassifier(classifiers=[clf1, clf2, clf3], use_probas=True, meta_classifier=lr, random_state=42) print('3-fold cross validation:\\n') for clf, label in zip([clf1, clf2, clf3, sclf], ['KNN', 'Random Forest', 'Naive Bayes', 'StackingClassifier']): scores = model_selection.cross_val_score(clf, X, y, cv=3, scoring='accuracy') print(\"Accuracy: %0.2f (+/- %0.2f) [%s]\" % (scores.mean(), scores.std(), label)) 3-fold cross validation: Accuracy: 0.91 (+/- 0.01) [KNN] Accuracy: 0.95 (+/- 0.01) [Random Forest] Accuracy: 0.91 (+/- 0.02) [Naive Bayes] Accuracy: 0.95 (+/- 0.02) [StackingClassifier]","title":"Example 2 - Using Probabilities as Meta-Features"},{"location":"user_guide/classifier/StackingCVClassifier/#example-3-stacked-cv-classification-and-gridsearch","text":"The stack allows tuning hyper parameters of the base and meta models! A full list of tunable parameters can be obtained via estimator.get_params().keys() . from sklearn.linear_model import LogisticRegression from sklearn.neighbors import KNeighborsClassifier from sklearn.naive_bayes import GaussianNB from sklearn.ensemble import RandomForestClassifier from sklearn.model_selection import GridSearchCV from mlxtend.classifier import StackingCVClassifier # Initializing models clf1 = KNeighborsClassifier(n_neighbors=1) clf2 = RandomForestClassifier(random_state=RANDOM_SEED) clf3 = GaussianNB() lr = LogisticRegression() sclf = StackingCVClassifier(classifiers=[clf1, clf2, clf3], meta_classifier=lr, random_state=42) params = {'kneighborsclassifier__n_neighbors': [1, 5], 'randomforestclassifier__n_estimators': [10, 50], 'meta_classifier__C': [0.1, 10.0]} grid = GridSearchCV(estimator=sclf, param_grid=params, cv=5, refit=True) grid.fit(X, y) cv_keys = ('mean_test_score', 'std_test_score', 'params') for r, _ in enumerate(grid.cv_results_['mean_test_score']): print(\"%0.3f +/- %0.2f %r\" % (grid.cv_results_[cv_keys[0]][r], grid.cv_results_[cv_keys[1]][r] / 2.0, grid.cv_results_[cv_keys[2]][r])) print('Best parameters: %s' % grid.best_params_) print('Accuracy: %.2f' % grid.best_score_) 0.947 +/- 0.03 {'kneighborsclassifier__n_neighbors': 1, 'meta_classifier__C': 0.1, 'randomforestclassifier__n_estimators': 10} 0.933 +/- 0.02 {'kneighborsclassifier__n_neighbors': 1, 'meta_classifier__C': 0.1, 'randomforestclassifier__n_estimators': 50} 0.940 +/- 0.02 {'kneighborsclassifier__n_neighbors': 1, 'meta_classifier__C': 10.0, 'randomforestclassifier__n_estimators': 10} 0.940 +/- 0.02 {'kneighborsclassifier__n_neighbors': 1, 'meta_classifier__C': 10.0, 'randomforestclassifier__n_estimators': 50} 0.953 +/- 0.02 {'kneighborsclassifier__n_neighbors': 5, 'meta_classifier__C': 0.1, 'randomforestclassifier__n_estimators': 10} 0.953 +/- 0.02 {'kneighborsclassifier__n_neighbors': 5, 'meta_classifier__C': 0.1, 'randomforestclassifier__n_estimators': 50} 0.953 +/- 0.02 {'kneighborsclassifier__n_neighbors': 5, 'meta_classifier__C': 10.0, 'randomforestclassifier__n_estimators': 10} 0.953 +/- 0.02 {'kneighborsclassifier__n_neighbors': 5, 'meta_classifier__C': 10.0, 'randomforestclassifier__n_estimators': 50} Best parameters: {'kneighborsclassifier__n_neighbors': 5, 'meta_classifier__C': 0.1, 'randomforestclassifier__n_estimators': 10} Accuracy: 0.95 In case we are planning to use a regression algorithm multiple times, all we need to do is to add an additional number suffix in the parameter grid as shown below: from sklearn.model_selection import GridSearchCV # Initializing models clf1 = KNeighborsClassifier(n_neighbors=1) clf2 = RandomForestClassifier(random_state=RANDOM_SEED) clf3 = GaussianNB() lr = LogisticRegression() sclf = StackingCVClassifier(classifiers=[clf1, clf1, clf2, clf3], meta_classifier=lr, random_state=RANDOM_SEED) params = {'kneighborsclassifier-1__n_neighbors': [1, 5], 'kneighborsclassifier-2__n_neighbors': [1, 5], 'randomforestclassifier__n_estimators': [10, 50], 'meta_classifier__C': [0.1, 10.0]} grid = GridSearchCV(estimator=sclf, param_grid=params, cv=5, refit=True) grid.fit(X, y) cv_keys = ('mean_test_score', 'std_test_score', 'params') for r, _ in enumerate(grid.cv_results_['mean_test_score']): print(\"%0.3f +/- %0.2f %r\" % (grid.cv_results_[cv_keys[0]][r], grid.cv_results_[cv_keys[1]][r] / 2.0, grid.cv_results_[cv_keys[2]][r])) print('Best parameters: %s' % grid.best_params_) print('Accuracy: %.2f' % grid.best_score_) 0.940 +/- 0.02 {'kneighborsclassifier-1__n_neighbors': 1, 'kneighborsclassifier-2__n_neighbors': 1, 'meta_classifier__C': 0.1, 'randomforestclassifier__n_estimators': 10} 0.940 +/- 0.02 {'kneighborsclassifier-1__n_neighbors': 1, 'kneighborsclassifier-2__n_neighbors': 1, 'meta_classifier__C': 0.1, 'randomforestclassifier__n_estimators': 50} 0.940 +/- 0.02 {'kneighborsclassifier-1__n_neighbors': 1, 'kneighborsclassifier-2__n_neighbors': 1, 'meta_classifier__C': 10.0, 'randomforestclassifier__n_estimators': 10} 0.940 +/- 0.02 {'kneighborsclassifier-1__n_neighbors': 1, 'kneighborsclassifier-2__n_neighbors': 1, 'meta_classifier__C': 10.0, 'randomforestclassifier__n_estimators': 50} 0.960 +/- 0.02 {'kneighborsclassifier-1__n_neighbors': 1, 'kneighborsclassifier-2__n_neighbors': 5, 'meta_classifier__C': 0.1, 'randomforestclassifier__n_estimators': 10} 0.953 +/- 0.02 {'kneighborsclassifier-1__n_neighbors': 1, 'kneighborsclassifier-2__n_neighbors': 5, 'meta_classifier__C': 0.1, 'randomforestclassifier__n_estimators': 50} 0.953 +/- 0.02 {'kneighborsclassifier-1__n_neighbors': 1, 'kneighborsclassifier-2__n_neighbors': 5, 'meta_classifier__C': 10.0, 'randomforestclassifier__n_estimators': 10} 0.953 +/- 0.02 {'kneighborsclassifier-1__n_neighbors': 1, 'kneighborsclassifier-2__n_neighbors': 5, 'meta_classifier__C': 10.0, 'randomforestclassifier__n_estimators': 50} 0.960 +/- 0.02 {'kneighborsclassifier-1__n_neighbors': 5, 'kneighborsclassifier-2__n_neighbors': 1, 'meta_classifier__C': 0.1, 'randomforestclassifier__n_estimators': 10} 0.953 +/- 0.02 {'kneighborsclassifier-1__n_neighbors': 5, 'kneighborsclassifier-2__n_neighbors': 1, 'meta_classifier__C': 0.1, 'randomforestclassifier__n_estimators': 50} 0.953 +/- 0.02 {'kneighborsclassifier-1__n_neighbors': 5, 'kneighborsclassifier-2__n_neighbors': 1, 'meta_classifier__C': 10.0, 'randomforestclassifier__n_estimators': 10} 0.953 +/- 0.02 {'kneighborsclassifier-1__n_neighbors': 5, 'kneighborsclassifier-2__n_neighbors': 1, 'meta_classifier__C': 10.0, 'randomforestclassifier__n_estimators': 50} 0.953 +/- 0.02 {'kneighborsclassifier-1__n_neighbors': 5, 'kneighborsclassifier-2__n_neighbors': 5, 'meta_classifier__C': 0.1, 'randomforestclassifier__n_estimators': 10} 0.953 +/- 0.02 {'kneighborsclassifier-1__n_neighbors': 5, 'kneighborsclassifier-2__n_neighbors': 5, 'meta_classifier__C': 0.1, 'randomforestclassifier__n_estimators': 50} 0.953 +/- 0.02 {'kneighborsclassifier-1__n_neighbors': 5, 'kneighborsclassifier-2__n_neighbors': 5, 'meta_classifier__C': 10.0, 'randomforestclassifier__n_estimators': 10} 0.953 +/- 0.02 {'kneighborsclassifier-1__n_neighbors': 5, 'kneighborsclassifier-2__n_neighbors': 5, 'meta_classifier__C': 10.0, 'randomforestclassifier__n_estimators': 50} Best parameters: {'kneighborsclassifier-1__n_neighbors': 1, 'kneighborsclassifier-2__n_neighbors': 5, 'meta_classifier__C': 0.1, 'randomforestclassifier__n_estimators': 10} Accuracy: 0.96 Note The StackingClassifier also enables grid search over the classifiers argument. When there are level-mixed hyperparameters, GridSearchCV will try to replace hyperparameters in a top-down order, i.e., classifers -> single base classifier -> classifier hyperparameter. For instance, given a hyperparameter grid such as params = {'randomforestclassifier__n_estimators': [1, 100], 'classifiers': [(clf1, clf1, clf1), (clf2, clf3)]} it will first use the instance settings of either (clf1, clf1, clf1) or (clf2, clf3). Then it will replace the 'n_estimators' settings for a matching classifier based on 'randomforestclassifier__n_estimators': [1, 100] .","title":"Example 3 - Stacked CV Classification and GridSearch"},{"location":"user_guide/classifier/StackingCVClassifier/#example-4-stacking-of-classifiers-that-operate-on-different-feature-subsets","text":"The different level-1 classifiers can be fit to different subsets of features in the training dataset. The following example illustrates how this can be done on a technical level using scikit-learn pipelines and the ColumnSelector : from sklearn.datasets import load_iris from mlxtend.classifier import StackingCVClassifier from mlxtend.feature_selection import ColumnSelector from sklearn.pipeline import make_pipeline from sklearn.linear_model import LogisticRegression iris = load_iris() X = iris.data y = iris.target pipe1 = make_pipeline(ColumnSelector(cols=(0, 2)), LogisticRegression()) pipe2 = make_pipeline(ColumnSelector(cols=(1, 2, 3)), LogisticRegression()) sclf = StackingCVClassifier(classifiers=[pipe1, pipe2], meta_classifier=LogisticRegression(), random_state=42) sclf.fit(X, y) StackingCVClassifier(classifiers=[Pipeline(memory=None, steps=[('columnselector', ColumnSelector(cols=(0, 2), drop_axis=False)), ('logisticregression', LogisticRegression(C=1.0, class_weight=None, dual=False, fit_intercept=True, intercept_scaling=1, l1_ratio=None, max_iter=100, multi_class='auto', n_jobs=None, penalty='l2', random_state=None, solver='lbfgs', tol=0.0... fit_intercept=True, intercept_scaling=1, l1_ratio=None, max_iter=100, multi_class='auto', n_jobs=None, penalty='l2', random_state=None, solver='lbfgs', tol=0.0001, verbose=0, warm_start=False), n_jobs=None, pre_dispatch='2*n_jobs', random_state=42, shuffle=True, store_train_meta_features=False, stratify=True, use_clones=True, use_features_in_secondary=False, use_probas=False, verbose=0)","title":"Example 4 - Stacking of Classifiers that Operate on Different Feature Subsets"},{"location":"user_guide/classifier/StackingCVClassifier/#example-5-roc-curve-with-decision_function","text":"Like other scikit-learn classifiers, the StackingCVClassifier has an decision_function method that can be used for plotting ROC curves. Note that the decision_function expects and requires the meta-classifier to implement a decision_function . from sklearn import model_selection from sklearn.linear_model import LogisticRegression from sklearn.neighbors import KNeighborsClassifier from sklearn.svm import SVC from sklearn.ensemble import RandomForestClassifier from mlxtend.classifier import StackingCVClassifier from sklearn.metrics import roc_curve, auc import numpy as np from sklearn.model_selection import train_test_split from sklearn import datasets from sklearn.preprocessing import label_binarize from sklearn.multiclass import OneVsRestClassifier iris = datasets.load_iris() X, y = iris.data[:, [0, 1]], iris.target # Binarize the output y = label_binarize(y, classes=[0, 1, 2]) n_classes = y.shape[1] RANDOM_SEED = 42 X_train, X_test, y_train, y_test = train_test_split( X, y, test_size=0.33, random_state=RANDOM_SEED) clf1 = LogisticRegression() clf2 = RandomForestClassifier(random_state=RANDOM_SEED) clf3 = SVC(random_state=RANDOM_SEED) lr = LogisticRegression() sclf = StackingCVClassifier(classifiers=[clf1, clf2, clf3], meta_classifier=lr) # Learn to predict each class against the other classifier = OneVsRestClassifier(sclf) Using predict_proba() y_score = classifier.fit(X_train, y_train).predict_proba(X_test) # Compute ROC curve and ROC area for each class fpr = dict() tpr = dict() roc_auc = dict() for i in range(n_classes): fpr[i], tpr[i], _ = roc_curve(y_test[:, i], y_score[:, i]) roc_auc[i] = auc(fpr[i], tpr[i]) # Compute micro-average ROC curve and ROC area fpr[\"micro\"], tpr[\"micro\"], _ = roc_curve(y_test.ravel(), y_score.ravel()) roc_auc[\"micro\"] = auc(fpr[\"micro\"], tpr[\"micro\"]) plt.figure() lw = 2 plt.plot(fpr[2], tpr[2], color='darkorange', lw=lw, label='ROC curve (area = %0.2f)' % roc_auc[2]) plt.plot([0, 1], [0, 1], color='navy', lw=lw, linestyle='--') plt.xlim([0.0, 1.0]) plt.ylim([0.0, 1.05]) plt.xlabel('False Positive Rate') plt.ylabel('True Positive Rate') plt.title('Receiver operating characteristic example') plt.legend(loc=\"lower right\") plt.show() Using decision_function() y_score = classifier.fit(X_train, y_train).decision_function(X_test) # Compute ROC curve and ROC area for each class fpr = dict() tpr = dict() roc_auc = dict() for i in range(n_classes): fpr[i], tpr[i], _ = roc_curve(y_test[:, i], y_score[:, i]) roc_auc[i] = auc(fpr[i], tpr[i]) # Compute micro-average ROC curve and ROC area fpr[\"micro\"], tpr[\"micro\"], _ = roc_curve(y_test.ravel(), y_score.ravel()) roc_auc[\"micro\"] = auc(fpr[\"micro\"], tpr[\"micro\"]) plt.figure() lw = 2 plt.plot(fpr[2], tpr[2], color='darkorange', lw=lw, label='ROC curve (area = %0.2f)' % roc_auc[2]) plt.plot([0, 1], [0, 1], color='navy', lw=lw, linestyle='--') plt.xlim([0.0, 1.0]) plt.ylim([0.0, 1.05]) plt.xlabel('False Positive Rate') plt.ylabel('True Positive Rate') plt.title('Receiver operating characteristic example') plt.legend(loc=\"lower right\") plt.show()","title":"Example 5 -- ROC Curve with decision_function"},{"location":"user_guide/classifier/StackingCVClassifier/#api","text":"StackingCVClassifier(classifiers, meta_classifier, use_probas=False, drop_proba_col=None, cv=2, shuffle=True, random_state=None, stratify=True, verbose=0, use_features_in_secondary=False, store_train_meta_features=False, use_clones=True, n_jobs=None, pre_dispatch='2 n_jobs')* A 'Stacking Cross-Validation' classifier for scikit-learn estimators. New in mlxtend v0.4.3 Parameters classifiers : array-like, shape = [n_classifiers] A list of classifiers. Invoking the fit method on the StackingCVClassifer will fit clones of these original classifiers that will be stored in the class attribute self.clfs_ . meta_classifier : object The meta-classifier to be fitted on the ensemble of classifiers use_probas : bool (default: False) If True, trains meta-classifier based on predicted probabilities instead of class labels. drop_proba_col : string (default: None) Drops extra \"probability\" column in the feature set, because it is redundant: p(y_c) = 1 - p(y_1) + p(y_2) + ... + p(y_{c-1}). This can be useful for meta-classifiers that are sensitive to perfectly collinear features. If last , drops last probability column. If first , drops first probability column. Only relevant if use_probas=True . cv : int, cross-validation generator or an iterable, optional (default: 2) Determines the cross-validation splitting strategy. Possible inputs for cv are: - None, to use the default 2-fold cross validation, - integer, to specify the number of folds in a (Stratified)KFold , - An object to be used as a cross-validation generator. - An iterable yielding train, test splits. For integer/None inputs, it will use either a KFold or StratifiedKFold cross validation depending the value of stratify argument. shuffle : bool (default: True) If True, and the cv argument is integer, the training data will be shuffled at fitting stage prior to cross-validation. If the cv argument is a specific cross validation technique, this argument is omitted. random_state : int, RandomState instance or None, optional (default: None) Constrols the randomness of the cv splitter. Used when cv is integer and shuffle=True . New in v0.16.0. stratify : bool (default: True) If True, and the cv argument is integer it will follow a stratified K-Fold cross validation technique. If the cv argument is a specific cross validation technique, this argument is omitted. verbose : int, optional (default=0) Controls the verbosity of the building process. - verbose=0 (default): Prints nothing - verbose=1 : Prints the number & name of the regressor being fitted and which fold is currently being used for fitting - verbose=2 : Prints info about the parameters of the regressor being fitted - verbose>2 : Changes verbose param of the underlying regressor to self.verbose - 2 use_features_in_secondary : bool (default: False) If True, the meta-classifier will be trained both on the predictions of the original classifiers and the original dataset. If False, the meta-classifier will be trained only on the predictions of the original classifiers. store_train_meta_features : bool (default: False) If True, the meta-features computed from the training data used for fitting the meta-classifier stored in the self.train_meta_features_ array, which can be accessed after calling fit . use_clones : bool (default: True) Clones the classifiers for stacking classification if True (default) or else uses the original ones, which will be refitted on the dataset upon calling the fit method. Hence, if use_clones=True, the original input classifiers will remain unmodified upon using the StackingCVClassifier's fit method. Setting use_clones=False is recommended if you are working with estimators that are supporting the scikit-learn fit/predict API interface but are not compatible to scikit-learn's clone function. n_jobs : int or None, optional (default=None) The number of CPUs to use to do the computation. None means 1 unless in a :obj: joblib.parallel_backend context. -1 means using all processors. See :term: Glossary <n_jobs> for more details. New in v0.16.0. pre_dispatch : int, or string, optional Controls the number of jobs that get dispatched during parallel execution. Reducing this number can be useful to avoid an explosion of memory consumption when more jobs get dispatched than CPUs can process. This parameter can be: - None, in which case all the jobs are immediately created and spawned. Use this for lightweight and fast-running jobs, to avoid delays due to on-demand spawning of the jobs - An int, giving the exact number of total jobs that are spawned - A string, giving an expression as a function of n_jobs, as in '2*n_jobs' New in v0.16.0. Attributes clfs_ : list, shape=[n_classifiers] Fitted classifiers (clones of the original classifiers) meta_clf_ : estimator Fitted meta-classifier (clone of the original meta-estimator) train_meta_features : numpy array, shape = [n_samples, n_classifiers] meta-features for training data, where n_samples is the number of samples in training data and n_classifiers is the number of classfiers. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/classifier/StackingCVClassifier/","title":"API"},{"location":"user_guide/classifier/StackingCVClassifier/#methods","text":"decision_function(X) Predict class confidence scores for X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns scores : shape=(n_samples,) if n_classes == 2 else (n_samples, n_classes). Confidence scores per (sample, class) combination. In the binary case, confidence score for self.classes_[1] where >0 means this class would be predicted. fit(X, y, groups=None, sample_weight=None) Fit ensemble classifers and the meta-classifier. Parameters X : numpy array, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : numpy array, shape = [n_samples] Target values. groups : numpy array/None, shape = [n_samples] The group that each sample belongs to. This is used by specific folding strategies such as GroupKFold() sample_weight : array-like, shape = [n_samples], optional Sample weights passed as sample_weights to each regressor in the regressors list as well as the meta_regressor. Raises error if some regressor does not support sample_weight in the fit() method. Returns self : object fit_transform(X, y=None, fit_params) Fit to data, then transform it. Fits transformer to X and y with optional parameters fit_params and returns a transformed version of X. Parameters X : numpy array of shape [n_samples, n_features] Training set. y : numpy array of shape [n_samples] Target values. **fit_params : dict Additional fit parameters. Returns X_new : numpy array of shape [n_samples, n_features_new] Transformed array. get_params(deep=True) Return estimator parameter names for GridSearch support. predict(X) Predict target values for X. Parameters X : numpy array, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns labels : array-like, shape = [n_samples] Predicted class labels. predict_meta_features(X) Get meta-features of test-data. Parameters X : numpy array, shape = [n_samples, n_features] Test vectors, where n_samples is the number of samples and n_features is the number of features. Returns meta-features : numpy array, shape = [n_samples, n_classifiers] Returns the meta-features for test data. predict_proba(X) Predict class probabilities for X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns proba : array-like, shape = [n_samples, n_classes] or a list of n_outputs of such arrays if n_outputs > 1. Probability for each class per sample. score(X, y, sample_weight=None) Return the mean accuracy on the given test data and labels. In multi-label classification, this is the subset accuracy which is a harsh metric since you require for each sample that each label set be correctly predicted. Parameters X : array-like of shape (n_samples, n_features) Test samples. y : array-like of shape (n_samples,) or (n_samples, n_outputs) True labels for X. sample_weight : array-like of shape (n_samples,), default=None Sample weights. Returns score : float Mean accuracy of self.predict(X) wrt. y. set_params( params) Set the parameters of this estimator. Valid parameter keys can be listed with get_params() . Returns self","title":"Methods"},{"location":"user_guide/classifier/StackingCVClassifier/#properties","text":"named_classifiers None","title":"Properties"},{"location":"user_guide/classifier/StackingClassifier/","text":"StackingClassifier: Simple stacking An ensemble-learning meta-classifier for stacking. from mlxtend.classifier import StackingClassifier Overview Stacking is an ensemble learning technique to combine multiple classification models via a meta-classifier. The individual classification models are trained based on the complete training set; then, the meta-classifier is fitted based on the outputs -- meta-features -- of the individual classification models in the ensemble. The meta-classifier can either be trained on the predicted class labels or probabilities from the ensemble. The algorithm can be summarized as follows (source: [1]): Please note that this type of Stacking is prone to overfitting due to information leakage. The related StackingCVClassifier.md does not derive the predictions for the 2nd-level classifier from the same datast that was used for training the level-1 classifiers and is recommended instead. References [1] Tang, J., S. Alelyani, and H. Liu. \" Data Classification: Algorithms and Applications. \" Data Mining and Knowledge Discovery Series, CRC Press (2015): pp. 498-500. [2] Wolpert, David H. \" Stacked generalization. \" Neural networks 5.2 (1992): 241-259. [3] Marios Michailidis (2017), StackNet, StackNet Meta Modelling Framework, https://github.com/kaz-Anova/StackNet Example 1 - Simple Stacked Classification from sklearn import datasets iris = datasets.load_iris() X, y = iris.data[:, 1:3], iris.target from sklearn import model_selection from sklearn.linear_model import LogisticRegression from sklearn.neighbors import KNeighborsClassifier from sklearn.naive_bayes import GaussianNB from sklearn.ensemble import RandomForestClassifier from mlxtend.classifier import StackingClassifier import numpy as np import warnings warnings.simplefilter('ignore') clf1 = KNeighborsClassifier(n_neighbors=1) clf2 = RandomForestClassifier(random_state=1) clf3 = GaussianNB() lr = LogisticRegression() sclf = StackingClassifier(classifiers=[clf1, clf2, clf3], meta_classifier=lr) print('3-fold cross validation:\\n') for clf, label in zip([clf1, clf2, clf3, sclf], ['KNN', 'Random Forest', 'Naive Bayes', 'StackingClassifier']): scores = model_selection.cross_val_score(clf, X, y, cv=3, scoring='accuracy') print(\"Accuracy: %0.2f (+/- %0.2f) [%s]\" % (scores.mean(), scores.std(), label)) 3-fold cross validation: Accuracy: 0.91 (+/- 0.01) [KNN] Accuracy: 0.95 (+/- 0.01) [Random Forest] Accuracy: 0.91 (+/- 0.02) [Naive Bayes] Accuracy: 0.95 (+/- 0.02) [StackingClassifier] import matplotlib.pyplot as plt from mlxtend.plotting import plot_decision_regions import matplotlib.gridspec as gridspec import itertools gs = gridspec.GridSpec(2, 2) fig = plt.figure(figsize=(10,8)) for clf, lab, grd in zip([clf1, clf2, clf3, sclf], ['KNN', 'Random Forest', 'Naive Bayes', 'StackingClassifier'], itertools.product([0, 1], repeat=2)): clf.fit(X, y) ax = plt.subplot(gs[grd[0], grd[1]]) fig = plot_decision_regions(X=X, y=y, clf=clf) plt.title(lab) Example 2 - Using Probabilities as Meta-Features Alternatively, the class-probabilities of the first-level classifiers can be used to train the meta-classifier (2nd-level classifier) by setting use_probas=True . If average_probas=True , the probabilities of the level-1 classifiers are averaged, if average_probas=False , the probabilities are stacked (recommended). For example, in a 3-class setting with 2 level-1 classifiers, these classifiers may make the following \"probability\" predictions for 1 training sample: classifier 1: [0.2, 0.5, 0.3] classifier 2: [0.3, 0.4, 0.4] If average_probas=True , the meta-features would be: [0.25, 0.45, 0.35] In contrast, using average_probas=False results in k features where, k = [n_classes * n_classifiers], by stacking these level-1 probabilities: [0.2, 0.5, 0.3, 0.3, 0.4, 0.4] clf1 = KNeighborsClassifier(n_neighbors=1) clf2 = RandomForestClassifier(random_state=1) clf3 = GaussianNB() lr = LogisticRegression() sclf = StackingClassifier(classifiers=[clf1, clf2, clf3], use_probas=True, average_probas=False, meta_classifier=lr) print('3-fold cross validation:\\n') for clf, label in zip([clf1, clf2, clf3, sclf], ['KNN', 'Random Forest', 'Naive Bayes', 'StackingClassifier']): scores = model_selection.cross_val_score(clf, X, y, cv=3, scoring='accuracy') print(\"Accuracy: %0.2f (+/- %0.2f) [%s]\" % (scores.mean(), scores.std(), label)) 3-fold cross validation: Accuracy: 0.91 (+/- 0.01) [KNN] Accuracy: 0.95 (+/- 0.01) [Random Forest] Accuracy: 0.91 (+/- 0.02) [Naive Bayes] Accuracy: 0.92 (+/- 0.02) [StackingClassifier] Example 3 - Stacked Classification and GridSearch The stack allows tuning hyper parameters of the base and meta models! A full list of tunable parameters can be obtained via estimator.get_params().keys() . from sklearn.linear_model import LogisticRegression from sklearn.neighbors import KNeighborsClassifier from sklearn.naive_bayes import GaussianNB from sklearn.ensemble import RandomForestClassifier from sklearn.model_selection import GridSearchCV from mlxtend.classifier import StackingClassifier # Initializing models clf1 = KNeighborsClassifier(n_neighbors=1) clf2 = RandomForestClassifier(random_state=1) clf3 = GaussianNB() lr = LogisticRegression() sclf = StackingClassifier(classifiers=[clf1, clf2, clf3], meta_classifier=lr) params = {'kneighborsclassifier__n_neighbors': [1, 5], 'randomforestclassifier__n_estimators': [10, 50], 'meta_classifier__C': [0.1, 10.0]} grid = GridSearchCV(estimator=sclf, param_grid=params, cv=5, refit=True) grid.fit(X, y) cv_keys = ('mean_test_score', 'std_test_score', 'params') for r, _ in enumerate(grid.cv_results_['mean_test_score']): print(\"%0.3f +/- %0.2f %r\" % (grid.cv_results_[cv_keys[0]][r], grid.cv_results_[cv_keys[1]][r] / 2.0, grid.cv_results_[cv_keys[2]][r])) print('Best parameters: %s' % grid.best_params_) print('Accuracy: %.2f' % grid.best_score_) 0.933 +/- 0.03 {'kneighborsclassifier__n_neighbors': 1, 'meta_classifier__C': 0.1, 'randomforestclassifier__n_estimators': 10} 0.940 +/- 0.02 {'kneighborsclassifier__n_neighbors': 1, 'meta_classifier__C': 0.1, 'randomforestclassifier__n_estimators': 50} 0.927 +/- 0.03 {'kneighborsclassifier__n_neighbors': 1, 'meta_classifier__C': 10.0, 'randomforestclassifier__n_estimators': 10} 0.947 +/- 0.02 {'kneighborsclassifier__n_neighbors': 1, 'meta_classifier__C': 10.0, 'randomforestclassifier__n_estimators': 50} 0.947 +/- 0.02 {'kneighborsclassifier__n_neighbors': 5, 'meta_classifier__C': 0.1, 'randomforestclassifier__n_estimators': 10} 0.947 +/- 0.02 {'kneighborsclassifier__n_neighbors': 5, 'meta_classifier__C': 0.1, 'randomforestclassifier__n_estimators': 50} 0.933 +/- 0.02 {'kneighborsclassifier__n_neighbors': 5, 'meta_classifier__C': 10.0, 'randomforestclassifier__n_estimators': 10} 0.940 +/- 0.02 {'kneighborsclassifier__n_neighbors': 5, 'meta_classifier__C': 10.0, 'randomforestclassifier__n_estimators': 50} Best parameters: {'kneighborsclassifier__n_neighbors': 1, 'meta_classifier__C': 10.0, 'randomforestclassifier__n_estimators': 50} Accuracy: 0.95 In case we are planning to use a regression algorithm multiple times, all we need to do is to add an additional number suffix in the parameter grid as shown below: from sklearn.model_selection import GridSearchCV # Initializing models clf1 = KNeighborsClassifier(n_neighbors=1) clf2 = RandomForestClassifier(random_state=1) clf3 = GaussianNB() lr = LogisticRegression() sclf = StackingClassifier(classifiers=[clf1, clf1, clf2, clf3], meta_classifier=lr) params = {'kneighborsclassifier-1__n_neighbors': [1, 5], 'kneighborsclassifier-2__n_neighbors': [1, 5], 'randomforestclassifier__n_estimators': [10, 50], 'meta_classifier__C': [0.1, 10.0]} grid = GridSearchCV(estimator=sclf, param_grid=params, cv=5, refit=True) grid.fit(X, y) cv_keys = ('mean_test_score', 'std_test_score', 'params') for r, _ in enumerate(grid.cv_results_['mean_test_score']): print(\"%0.3f +/- %0.2f %r\" % (grid.cv_results_[cv_keys[0]][r], grid.cv_results_[cv_keys[1]][r] / 2.0, grid.cv_results_[cv_keys[2]][r])) print('Best parameters: %s' % grid.best_params_) print('Accuracy: %.2f' % grid.best_score_) 0.933 +/- 0.03 {'kneighborsclassifier-1__n_neighbors': 1, 'kneighborsclassifier-2__n_neighbors': 1, 'meta_classifier__C': 0.1, 'randomforestclassifier__n_estimators': 10} 0.940 +/- 0.02 {'kneighborsclassifier-1__n_neighbors': 1, 'kneighborsclassifier-2__n_neighbors': 1, 'meta_classifier__C': 0.1, 'randomforestclassifier__n_estimators': 50} 0.927 +/- 0.03 {'kneighborsclassifier-1__n_neighbors': 1, 'kneighborsclassifier-2__n_neighbors': 1, 'meta_classifier__C': 10.0, 'randomforestclassifier__n_estimators': 10} 0.947 +/- 0.02 {'kneighborsclassifier-1__n_neighbors': 1, 'kneighborsclassifier-2__n_neighbors': 1, 'meta_classifier__C': 10.0, 'randomforestclassifier__n_estimators': 50} 0.940 +/- 0.02 {'kneighborsclassifier-1__n_neighbors': 1, 'kneighborsclassifier-2__n_neighbors': 5, 'meta_classifier__C': 0.1, 'randomforestclassifier__n_estimators': 10} 0.947 +/- 0.02 {'kneighborsclassifier-1__n_neighbors': 1, 'kneighborsclassifier-2__n_neighbors': 5, 'meta_classifier__C': 0.1, 'randomforestclassifier__n_estimators': 50} 0.933 +/- 0.03 {'kneighborsclassifier-1__n_neighbors': 1, 'kneighborsclassifier-2__n_neighbors': 5, 'meta_classifier__C': 10.0, 'randomforestclassifier__n_estimators': 10} 0.953 +/- 0.02 {'kneighborsclassifier-1__n_neighbors': 1, 'kneighborsclassifier-2__n_neighbors': 5, 'meta_classifier__C': 10.0, 'randomforestclassifier__n_estimators': 50} 0.940 +/- 0.02 {'kneighborsclassifier-1__n_neighbors': 5, 'kneighborsclassifier-2__n_neighbors': 1, 'meta_classifier__C': 0.1, 'randomforestclassifier__n_estimators': 10} 0.947 +/- 0.02 {'kneighborsclassifier-1__n_neighbors': 5, 'kneighborsclassifier-2__n_neighbors': 1, 'meta_classifier__C': 0.1, 'randomforestclassifier__n_estimators': 50} 0.933 +/- 0.03 {'kneighborsclassifier-1__n_neighbors': 5, 'kneighborsclassifier-2__n_neighbors': 1, 'meta_classifier__C': 10.0, 'randomforestclassifier__n_estimators': 10} 0.953 +/- 0.02 {'kneighborsclassifier-1__n_neighbors': 5, 'kneighborsclassifier-2__n_neighbors': 1, 'meta_classifier__C': 10.0, 'randomforestclassifier__n_estimators': 50} 0.953 +/- 0.02 {'kneighborsclassifier-1__n_neighbors': 5, 'kneighborsclassifier-2__n_neighbors': 5, 'meta_classifier__C': 0.1, 'randomforestclassifier__n_estimators': 10} 0.953 +/- 0.02 {'kneighborsclassifier-1__n_neighbors': 5, 'kneighborsclassifier-2__n_neighbors': 5, 'meta_classifier__C': 0.1, 'randomforestclassifier__n_estimators': 50} 0.933 +/- 0.02 {'kneighborsclassifier-1__n_neighbors': 5, 'kneighborsclassifier-2__n_neighbors': 5, 'meta_classifier__C': 10.0, 'randomforestclassifier__n_estimators': 10} 0.940 +/- 0.02 {'kneighborsclassifier-1__n_neighbors': 5, 'kneighborsclassifier-2__n_neighbors': 5, 'meta_classifier__C': 10.0, 'randomforestclassifier__n_estimators': 50} Best parameters: {'kneighborsclassifier-1__n_neighbors': 1, 'kneighborsclassifier-2__n_neighbors': 5, 'meta_classifier__C': 10.0, 'randomforestclassifier__n_estimators': 50} Accuracy: 0.95 Note The StackingClassifier also enables grid search over the classifiers argument. When there are level-mixed hyperparameters, GridSearchCV will try to replace hyperparameters in a top-down order, i.e., classifers -> single base classifier -> classifier hyperparameter. For instance, given a hyperparameter grid such as params = {'randomforestclassifier__n_estimators': [1, 100], 'classifiers': [(clf1, clf1, clf1), (clf2, clf3)]} it will first use the instance settings of either (clf1, clf1, clf1) or (clf2, clf3). Then it will replace the 'n_estimators' settings for a matching classifier based on 'randomforestclassifier__n_estimators': [1, 100] . Example 4 - Stacking of Classifiers that Operate on Different Feature Subsets The different level-1 classifiers can be fit to different subsets of features in the training dataset. The following example illustrates how this can be done on a technical level using scikit-learn pipelines and the ColumnSelector : from sklearn.datasets import load_iris from mlxtend.classifier import StackingClassifier from mlxtend.feature_selection import ColumnSelector from sklearn.pipeline import make_pipeline from sklearn.linear_model import LogisticRegression iris = load_iris() X = iris.data y = iris.target pipe1 = make_pipeline(ColumnSelector(cols=(0, 2)), LogisticRegression()) pipe2 = make_pipeline(ColumnSelector(cols=(1, 2, 3)), LogisticRegression()) sclf = StackingClassifier(classifiers=[pipe1, pipe2], meta_classifier=LogisticRegression()) sclf.fit(X, y) StackingClassifier(average_probas=False, classifiers=[Pipeline(memory=None, steps=[('columnselector', ColumnSelector(cols=(0, 2), drop_axis=False)), ('logisticregression', LogisticRegression(C=1.0, class_weight=None, dual=False, fit_intercept=True, intercept_scaling=1, l1_ratio=None, max_iter=100, multi_class='auto', n_jobs=None, penalty='l2', random_state=None, sol... meta_classifier=LogisticRegression(C=1.0, class_weight=None, dual=False, fit_intercept=True, intercept_scaling=1, l1_ratio=None, max_iter=100, multi_class='auto', n_jobs=None, penalty='l2', random_state=None, solver='lbfgs', tol=0.0001, verbose=0, warm_start=False), store_train_meta_features=False, use_clones=True, use_features_in_secondary=False, use_probas=False, verbose=0) Example 5 - Using Pre-fitted Classifiers Assume that we previously fitted our classifiers: from sklearn import model_selection from sklearn.linear_model import LogisticRegression from sklearn.naive_bayes import GaussianNB from sklearn.ensemble import RandomForestClassifier import numpy as np clf1 = KNeighborsClassifier(n_neighbors=1) clf2 = RandomForestClassifier(random_state=1) clf3 = GaussianNB() lr = LogisticRegression() for clf in (clf1, clf2, clf3): clf.fit(X, y) By setting fit_base_estimators=False , it will enforce use_clones to be False and the StackingClassifier will not re-fit these classifers to save computational time: from mlxtend.classifier import StackingClassifier import copy sclf = StackingClassifier(classifiers=[clf1, clf2, clf3], meta_classifier=lr, fit_base_estimators=False) labels = ['KNN', 'Random Forest', 'Naive Bayes', 'StackingClassifier'] sclf.fit(X, y) print('accuracy:', np.mean(y == sclf.predict(X))) Warning: enforce use_clones to be False accuracy: 1.0 However, please note that fit_base_estimators=False is incompatible to any form of cross-validation that is done in e.g., model_selection.cross_val_score or model_selection.GridSearchCV , etc., since it would require the classifiers to be refit to the training folds. Thus, only use fit_base_estimators=False if you want to make a prediction directly without cross-validation. Example 6 -- ROC Curve with decision_function Like other scikit-learn classifiers, the StackingCVClassifier has an decision_function method that can be used for plotting ROC curves. Note that the decision_function expects and requires the meta-classifier to implement a decision_function . from sklearn import model_selection from sklearn.linear_model import LogisticRegression from sklearn.neighbors import KNeighborsClassifier from sklearn.svm import SVC from sklearn.ensemble import RandomForestClassifier from mlxtend.classifier import StackingCVClassifier from sklearn.metrics import roc_curve, auc import numpy as np from sklearn.model_selection import train_test_split from sklearn import datasets from sklearn.preprocessing import label_binarize from sklearn.multiclass import OneVsRestClassifier iris = datasets.load_iris() X, y = iris.data[:, [0, 1]], iris.target # Binarize the output y = label_binarize(y, classes=[0, 1, 2]) n_classes = y.shape[1] RANDOM_SEED = 42 X_train, X_test, y_train, y_test = train_test_split( X, y, test_size=0.33, random_state=RANDOM_SEED) clf1 = LogisticRegression() clf2 = RandomForestClassifier(random_state=RANDOM_SEED) clf3 = SVC(random_state=RANDOM_SEED) lr = LogisticRegression() sclf = StackingClassifier(classifiers=[clf1, clf2, clf3], meta_classifier=lr) # Learn to predict each class against the other classifier = OneVsRestClassifier(sclf) Using predict_proba() y_score = classifier.fit(X_train, y_train).predict_proba(X_test) # Compute ROC curve and ROC area for each class fpr = dict() tpr = dict() roc_auc = dict() for i in range(n_classes): fpr[i], tpr[i], _ = roc_curve(y_test[:, i], y_score[:, i]) roc_auc[i] = auc(fpr[i], tpr[i]) # Compute micro-average ROC curve and ROC area fpr[\"micro\"], tpr[\"micro\"], _ = roc_curve(y_test.ravel(), y_score.ravel()) roc_auc[\"micro\"] = auc(fpr[\"micro\"], tpr[\"micro\"]) plt.figure() lw = 2 plt.plot(fpr[2], tpr[2], color='darkorange', lw=lw, label='ROC curve (area = %0.2f)' % roc_auc[2]) plt.plot([0, 1], [0, 1], color='navy', lw=lw, linestyle='--') plt.xlim([0.0, 1.0]) plt.ylim([0.0, 1.05]) plt.xlabel('False Positive Rate') plt.ylabel('True Positive Rate') plt.title('Receiver operating characteristic example') plt.legend(loc=\"lower right\") plt.show() Using decision_function() y_score = classifier.fit(X_train, y_train).decision_function(X_test) # Compute ROC curve and ROC area for each class fpr = dict() tpr = dict() roc_auc = dict() for i in range(n_classes): fpr[i], tpr[i], _ = roc_curve(y_test[:, i], y_score[:, i]) roc_auc[i] = auc(fpr[i], tpr[i]) # Compute micro-average ROC curve and ROC area fpr[\"micro\"], tpr[\"micro\"], _ = roc_curve(y_test.ravel(), y_score.ravel()) roc_auc[\"micro\"] = auc(fpr[\"micro\"], tpr[\"micro\"]) plt.figure() lw = 2 plt.plot(fpr[2], tpr[2], color='darkorange', lw=lw, label='ROC curve (area = %0.2f)' % roc_auc[2]) plt.plot([0, 1], [0, 1], color='navy', lw=lw, linestyle='--') plt.xlim([0.0, 1.0]) plt.ylim([0.0, 1.05]) plt.xlabel('False Positive Rate') plt.ylabel('True Positive Rate') plt.title('Receiver operating characteristic example') plt.legend(loc=\"lower right\") plt.show() API","title":"StackingClassifier: Simple stacking"},{"location":"user_guide/classifier/StackingClassifier/#stackingclassifier-simple-stacking","text":"An ensemble-learning meta-classifier for stacking. from mlxtend.classifier import StackingClassifier","title":"StackingClassifier: Simple stacking"},{"location":"user_guide/classifier/StackingClassifier/#overview","text":"Stacking is an ensemble learning technique to combine multiple classification models via a meta-classifier. The individual classification models are trained based on the complete training set; then, the meta-classifier is fitted based on the outputs -- meta-features -- of the individual classification models in the ensemble. The meta-classifier can either be trained on the predicted class labels or probabilities from the ensemble. The algorithm can be summarized as follows (source: [1]): Please note that this type of Stacking is prone to overfitting due to information leakage. The related StackingCVClassifier.md does not derive the predictions for the 2nd-level classifier from the same datast that was used for training the level-1 classifiers and is recommended instead.","title":"Overview"},{"location":"user_guide/classifier/StackingClassifier/#references","text":"[1] Tang, J., S. Alelyani, and H. Liu. \" Data Classification: Algorithms and Applications. \" Data Mining and Knowledge Discovery Series, CRC Press (2015): pp. 498-500. [2] Wolpert, David H. \" Stacked generalization. \" Neural networks 5.2 (1992): 241-259. [3] Marios Michailidis (2017), StackNet, StackNet Meta Modelling Framework, https://github.com/kaz-Anova/StackNet","title":"References"},{"location":"user_guide/classifier/StackingClassifier/#example-1-simple-stacked-classification","text":"from sklearn import datasets iris = datasets.load_iris() X, y = iris.data[:, 1:3], iris.target from sklearn import model_selection from sklearn.linear_model import LogisticRegression from sklearn.neighbors import KNeighborsClassifier from sklearn.naive_bayes import GaussianNB from sklearn.ensemble import RandomForestClassifier from mlxtend.classifier import StackingClassifier import numpy as np import warnings warnings.simplefilter('ignore') clf1 = KNeighborsClassifier(n_neighbors=1) clf2 = RandomForestClassifier(random_state=1) clf3 = GaussianNB() lr = LogisticRegression() sclf = StackingClassifier(classifiers=[clf1, clf2, clf3], meta_classifier=lr) print('3-fold cross validation:\\n') for clf, label in zip([clf1, clf2, clf3, sclf], ['KNN', 'Random Forest', 'Naive Bayes', 'StackingClassifier']): scores = model_selection.cross_val_score(clf, X, y, cv=3, scoring='accuracy') print(\"Accuracy: %0.2f (+/- %0.2f) [%s]\" % (scores.mean(), scores.std(), label)) 3-fold cross validation: Accuracy: 0.91 (+/- 0.01) [KNN] Accuracy: 0.95 (+/- 0.01) [Random Forest] Accuracy: 0.91 (+/- 0.02) [Naive Bayes] Accuracy: 0.95 (+/- 0.02) [StackingClassifier] import matplotlib.pyplot as plt from mlxtend.plotting import plot_decision_regions import matplotlib.gridspec as gridspec import itertools gs = gridspec.GridSpec(2, 2) fig = plt.figure(figsize=(10,8)) for clf, lab, grd in zip([clf1, clf2, clf3, sclf], ['KNN', 'Random Forest', 'Naive Bayes', 'StackingClassifier'], itertools.product([0, 1], repeat=2)): clf.fit(X, y) ax = plt.subplot(gs[grd[0], grd[1]]) fig = plot_decision_regions(X=X, y=y, clf=clf) plt.title(lab)","title":"Example 1 - Simple Stacked Classification"},{"location":"user_guide/classifier/StackingClassifier/#example-2-using-probabilities-as-meta-features","text":"Alternatively, the class-probabilities of the first-level classifiers can be used to train the meta-classifier (2nd-level classifier) by setting use_probas=True . If average_probas=True , the probabilities of the level-1 classifiers are averaged, if average_probas=False , the probabilities are stacked (recommended). For example, in a 3-class setting with 2 level-1 classifiers, these classifiers may make the following \"probability\" predictions for 1 training sample: classifier 1: [0.2, 0.5, 0.3] classifier 2: [0.3, 0.4, 0.4] If average_probas=True , the meta-features would be: [0.25, 0.45, 0.35] In contrast, using average_probas=False results in k features where, k = [n_classes * n_classifiers], by stacking these level-1 probabilities: [0.2, 0.5, 0.3, 0.3, 0.4, 0.4] clf1 = KNeighborsClassifier(n_neighbors=1) clf2 = RandomForestClassifier(random_state=1) clf3 = GaussianNB() lr = LogisticRegression() sclf = StackingClassifier(classifiers=[clf1, clf2, clf3], use_probas=True, average_probas=False, meta_classifier=lr) print('3-fold cross validation:\\n') for clf, label in zip([clf1, clf2, clf3, sclf], ['KNN', 'Random Forest', 'Naive Bayes', 'StackingClassifier']): scores = model_selection.cross_val_score(clf, X, y, cv=3, scoring='accuracy') print(\"Accuracy: %0.2f (+/- %0.2f) [%s]\" % (scores.mean(), scores.std(), label)) 3-fold cross validation: Accuracy: 0.91 (+/- 0.01) [KNN] Accuracy: 0.95 (+/- 0.01) [Random Forest] Accuracy: 0.91 (+/- 0.02) [Naive Bayes] Accuracy: 0.92 (+/- 0.02) [StackingClassifier]","title":"Example 2 - Using Probabilities as Meta-Features"},{"location":"user_guide/classifier/StackingClassifier/#example-3-stacked-classification-and-gridsearch","text":"The stack allows tuning hyper parameters of the base and meta models! A full list of tunable parameters can be obtained via estimator.get_params().keys() . from sklearn.linear_model import LogisticRegression from sklearn.neighbors import KNeighborsClassifier from sklearn.naive_bayes import GaussianNB from sklearn.ensemble import RandomForestClassifier from sklearn.model_selection import GridSearchCV from mlxtend.classifier import StackingClassifier # Initializing models clf1 = KNeighborsClassifier(n_neighbors=1) clf2 = RandomForestClassifier(random_state=1) clf3 = GaussianNB() lr = LogisticRegression() sclf = StackingClassifier(classifiers=[clf1, clf2, clf3], meta_classifier=lr) params = {'kneighborsclassifier__n_neighbors': [1, 5], 'randomforestclassifier__n_estimators': [10, 50], 'meta_classifier__C': [0.1, 10.0]} grid = GridSearchCV(estimator=sclf, param_grid=params, cv=5, refit=True) grid.fit(X, y) cv_keys = ('mean_test_score', 'std_test_score', 'params') for r, _ in enumerate(grid.cv_results_['mean_test_score']): print(\"%0.3f +/- %0.2f %r\" % (grid.cv_results_[cv_keys[0]][r], grid.cv_results_[cv_keys[1]][r] / 2.0, grid.cv_results_[cv_keys[2]][r])) print('Best parameters: %s' % grid.best_params_) print('Accuracy: %.2f' % grid.best_score_) 0.933 +/- 0.03 {'kneighborsclassifier__n_neighbors': 1, 'meta_classifier__C': 0.1, 'randomforestclassifier__n_estimators': 10} 0.940 +/- 0.02 {'kneighborsclassifier__n_neighbors': 1, 'meta_classifier__C': 0.1, 'randomforestclassifier__n_estimators': 50} 0.927 +/- 0.03 {'kneighborsclassifier__n_neighbors': 1, 'meta_classifier__C': 10.0, 'randomforestclassifier__n_estimators': 10} 0.947 +/- 0.02 {'kneighborsclassifier__n_neighbors': 1, 'meta_classifier__C': 10.0, 'randomforestclassifier__n_estimators': 50} 0.947 +/- 0.02 {'kneighborsclassifier__n_neighbors': 5, 'meta_classifier__C': 0.1, 'randomforestclassifier__n_estimators': 10} 0.947 +/- 0.02 {'kneighborsclassifier__n_neighbors': 5, 'meta_classifier__C': 0.1, 'randomforestclassifier__n_estimators': 50} 0.933 +/- 0.02 {'kneighborsclassifier__n_neighbors': 5, 'meta_classifier__C': 10.0, 'randomforestclassifier__n_estimators': 10} 0.940 +/- 0.02 {'kneighborsclassifier__n_neighbors': 5, 'meta_classifier__C': 10.0, 'randomforestclassifier__n_estimators': 50} Best parameters: {'kneighborsclassifier__n_neighbors': 1, 'meta_classifier__C': 10.0, 'randomforestclassifier__n_estimators': 50} Accuracy: 0.95 In case we are planning to use a regression algorithm multiple times, all we need to do is to add an additional number suffix in the parameter grid as shown below: from sklearn.model_selection import GridSearchCV # Initializing models clf1 = KNeighborsClassifier(n_neighbors=1) clf2 = RandomForestClassifier(random_state=1) clf3 = GaussianNB() lr = LogisticRegression() sclf = StackingClassifier(classifiers=[clf1, clf1, clf2, clf3], meta_classifier=lr) params = {'kneighborsclassifier-1__n_neighbors': [1, 5], 'kneighborsclassifier-2__n_neighbors': [1, 5], 'randomforestclassifier__n_estimators': [10, 50], 'meta_classifier__C': [0.1, 10.0]} grid = GridSearchCV(estimator=sclf, param_grid=params, cv=5, refit=True) grid.fit(X, y) cv_keys = ('mean_test_score', 'std_test_score', 'params') for r, _ in enumerate(grid.cv_results_['mean_test_score']): print(\"%0.3f +/- %0.2f %r\" % (grid.cv_results_[cv_keys[0]][r], grid.cv_results_[cv_keys[1]][r] / 2.0, grid.cv_results_[cv_keys[2]][r])) print('Best parameters: %s' % grid.best_params_) print('Accuracy: %.2f' % grid.best_score_) 0.933 +/- 0.03 {'kneighborsclassifier-1__n_neighbors': 1, 'kneighborsclassifier-2__n_neighbors': 1, 'meta_classifier__C': 0.1, 'randomforestclassifier__n_estimators': 10} 0.940 +/- 0.02 {'kneighborsclassifier-1__n_neighbors': 1, 'kneighborsclassifier-2__n_neighbors': 1, 'meta_classifier__C': 0.1, 'randomforestclassifier__n_estimators': 50} 0.927 +/- 0.03 {'kneighborsclassifier-1__n_neighbors': 1, 'kneighborsclassifier-2__n_neighbors': 1, 'meta_classifier__C': 10.0, 'randomforestclassifier__n_estimators': 10} 0.947 +/- 0.02 {'kneighborsclassifier-1__n_neighbors': 1, 'kneighborsclassifier-2__n_neighbors': 1, 'meta_classifier__C': 10.0, 'randomforestclassifier__n_estimators': 50} 0.940 +/- 0.02 {'kneighborsclassifier-1__n_neighbors': 1, 'kneighborsclassifier-2__n_neighbors': 5, 'meta_classifier__C': 0.1, 'randomforestclassifier__n_estimators': 10} 0.947 +/- 0.02 {'kneighborsclassifier-1__n_neighbors': 1, 'kneighborsclassifier-2__n_neighbors': 5, 'meta_classifier__C': 0.1, 'randomforestclassifier__n_estimators': 50} 0.933 +/- 0.03 {'kneighborsclassifier-1__n_neighbors': 1, 'kneighborsclassifier-2__n_neighbors': 5, 'meta_classifier__C': 10.0, 'randomforestclassifier__n_estimators': 10} 0.953 +/- 0.02 {'kneighborsclassifier-1__n_neighbors': 1, 'kneighborsclassifier-2__n_neighbors': 5, 'meta_classifier__C': 10.0, 'randomforestclassifier__n_estimators': 50} 0.940 +/- 0.02 {'kneighborsclassifier-1__n_neighbors': 5, 'kneighborsclassifier-2__n_neighbors': 1, 'meta_classifier__C': 0.1, 'randomforestclassifier__n_estimators': 10} 0.947 +/- 0.02 {'kneighborsclassifier-1__n_neighbors': 5, 'kneighborsclassifier-2__n_neighbors': 1, 'meta_classifier__C': 0.1, 'randomforestclassifier__n_estimators': 50} 0.933 +/- 0.03 {'kneighborsclassifier-1__n_neighbors': 5, 'kneighborsclassifier-2__n_neighbors': 1, 'meta_classifier__C': 10.0, 'randomforestclassifier__n_estimators': 10} 0.953 +/- 0.02 {'kneighborsclassifier-1__n_neighbors': 5, 'kneighborsclassifier-2__n_neighbors': 1, 'meta_classifier__C': 10.0, 'randomforestclassifier__n_estimators': 50} 0.953 +/- 0.02 {'kneighborsclassifier-1__n_neighbors': 5, 'kneighborsclassifier-2__n_neighbors': 5, 'meta_classifier__C': 0.1, 'randomforestclassifier__n_estimators': 10} 0.953 +/- 0.02 {'kneighborsclassifier-1__n_neighbors': 5, 'kneighborsclassifier-2__n_neighbors': 5, 'meta_classifier__C': 0.1, 'randomforestclassifier__n_estimators': 50} 0.933 +/- 0.02 {'kneighborsclassifier-1__n_neighbors': 5, 'kneighborsclassifier-2__n_neighbors': 5, 'meta_classifier__C': 10.0, 'randomforestclassifier__n_estimators': 10} 0.940 +/- 0.02 {'kneighborsclassifier-1__n_neighbors': 5, 'kneighborsclassifier-2__n_neighbors': 5, 'meta_classifier__C': 10.0, 'randomforestclassifier__n_estimators': 50} Best parameters: {'kneighborsclassifier-1__n_neighbors': 1, 'kneighborsclassifier-2__n_neighbors': 5, 'meta_classifier__C': 10.0, 'randomforestclassifier__n_estimators': 50} Accuracy: 0.95 Note The StackingClassifier also enables grid search over the classifiers argument. When there are level-mixed hyperparameters, GridSearchCV will try to replace hyperparameters in a top-down order, i.e., classifers -> single base classifier -> classifier hyperparameter. For instance, given a hyperparameter grid such as params = {'randomforestclassifier__n_estimators': [1, 100], 'classifiers': [(clf1, clf1, clf1), (clf2, clf3)]} it will first use the instance settings of either (clf1, clf1, clf1) or (clf2, clf3). Then it will replace the 'n_estimators' settings for a matching classifier based on 'randomforestclassifier__n_estimators': [1, 100] .","title":"Example 3 - Stacked Classification and GridSearch"},{"location":"user_guide/classifier/StackingClassifier/#example-4-stacking-of-classifiers-that-operate-on-different-feature-subsets","text":"The different level-1 classifiers can be fit to different subsets of features in the training dataset. The following example illustrates how this can be done on a technical level using scikit-learn pipelines and the ColumnSelector : from sklearn.datasets import load_iris from mlxtend.classifier import StackingClassifier from mlxtend.feature_selection import ColumnSelector from sklearn.pipeline import make_pipeline from sklearn.linear_model import LogisticRegression iris = load_iris() X = iris.data y = iris.target pipe1 = make_pipeline(ColumnSelector(cols=(0, 2)), LogisticRegression()) pipe2 = make_pipeline(ColumnSelector(cols=(1, 2, 3)), LogisticRegression()) sclf = StackingClassifier(classifiers=[pipe1, pipe2], meta_classifier=LogisticRegression()) sclf.fit(X, y) StackingClassifier(average_probas=False, classifiers=[Pipeline(memory=None, steps=[('columnselector', ColumnSelector(cols=(0, 2), drop_axis=False)), ('logisticregression', LogisticRegression(C=1.0, class_weight=None, dual=False, fit_intercept=True, intercept_scaling=1, l1_ratio=None, max_iter=100, multi_class='auto', n_jobs=None, penalty='l2', random_state=None, sol... meta_classifier=LogisticRegression(C=1.0, class_weight=None, dual=False, fit_intercept=True, intercept_scaling=1, l1_ratio=None, max_iter=100, multi_class='auto', n_jobs=None, penalty='l2', random_state=None, solver='lbfgs', tol=0.0001, verbose=0, warm_start=False), store_train_meta_features=False, use_clones=True, use_features_in_secondary=False, use_probas=False, verbose=0)","title":"Example 4 - Stacking of Classifiers that Operate on Different Feature Subsets"},{"location":"user_guide/classifier/StackingClassifier/#example-5-using-pre-fitted-classifiers","text":"Assume that we previously fitted our classifiers: from sklearn import model_selection from sklearn.linear_model import LogisticRegression from sklearn.naive_bayes import GaussianNB from sklearn.ensemble import RandomForestClassifier import numpy as np clf1 = KNeighborsClassifier(n_neighbors=1) clf2 = RandomForestClassifier(random_state=1) clf3 = GaussianNB() lr = LogisticRegression() for clf in (clf1, clf2, clf3): clf.fit(X, y) By setting fit_base_estimators=False , it will enforce use_clones to be False and the StackingClassifier will not re-fit these classifers to save computational time: from mlxtend.classifier import StackingClassifier import copy sclf = StackingClassifier(classifiers=[clf1, clf2, clf3], meta_classifier=lr, fit_base_estimators=False) labels = ['KNN', 'Random Forest', 'Naive Bayes', 'StackingClassifier'] sclf.fit(X, y) print('accuracy:', np.mean(y == sclf.predict(X))) Warning: enforce use_clones to be False accuracy: 1.0 However, please note that fit_base_estimators=False is incompatible to any form of cross-validation that is done in e.g., model_selection.cross_val_score or model_selection.GridSearchCV , etc., since it would require the classifiers to be refit to the training folds. Thus, only use fit_base_estimators=False if you want to make a prediction directly without cross-validation.","title":"Example 5 - Using Pre-fitted Classifiers"},{"location":"user_guide/classifier/StackingClassifier/#example-6-roc-curve-with-decision_function","text":"Like other scikit-learn classifiers, the StackingCVClassifier has an decision_function method that can be used for plotting ROC curves. Note that the decision_function expects and requires the meta-classifier to implement a decision_function . from sklearn import model_selection from sklearn.linear_model import LogisticRegression from sklearn.neighbors import KNeighborsClassifier from sklearn.svm import SVC from sklearn.ensemble import RandomForestClassifier from mlxtend.classifier import StackingCVClassifier from sklearn.metrics import roc_curve, auc import numpy as np from sklearn.model_selection import train_test_split from sklearn import datasets from sklearn.preprocessing import label_binarize from sklearn.multiclass import OneVsRestClassifier iris = datasets.load_iris() X, y = iris.data[:, [0, 1]], iris.target # Binarize the output y = label_binarize(y, classes=[0, 1, 2]) n_classes = y.shape[1] RANDOM_SEED = 42 X_train, X_test, y_train, y_test = train_test_split( X, y, test_size=0.33, random_state=RANDOM_SEED) clf1 = LogisticRegression() clf2 = RandomForestClassifier(random_state=RANDOM_SEED) clf3 = SVC(random_state=RANDOM_SEED) lr = LogisticRegression() sclf = StackingClassifier(classifiers=[clf1, clf2, clf3], meta_classifier=lr) # Learn to predict each class against the other classifier = OneVsRestClassifier(sclf) Using predict_proba() y_score = classifier.fit(X_train, y_train).predict_proba(X_test) # Compute ROC curve and ROC area for each class fpr = dict() tpr = dict() roc_auc = dict() for i in range(n_classes): fpr[i], tpr[i], _ = roc_curve(y_test[:, i], y_score[:, i]) roc_auc[i] = auc(fpr[i], tpr[i]) # Compute micro-average ROC curve and ROC area fpr[\"micro\"], tpr[\"micro\"], _ = roc_curve(y_test.ravel(), y_score.ravel()) roc_auc[\"micro\"] = auc(fpr[\"micro\"], tpr[\"micro\"]) plt.figure() lw = 2 plt.plot(fpr[2], tpr[2], color='darkorange', lw=lw, label='ROC curve (area = %0.2f)' % roc_auc[2]) plt.plot([0, 1], [0, 1], color='navy', lw=lw, linestyle='--') plt.xlim([0.0, 1.0]) plt.ylim([0.0, 1.05]) plt.xlabel('False Positive Rate') plt.ylabel('True Positive Rate') plt.title('Receiver operating characteristic example') plt.legend(loc=\"lower right\") plt.show() Using decision_function() y_score = classifier.fit(X_train, y_train).decision_function(X_test) # Compute ROC curve and ROC area for each class fpr = dict() tpr = dict() roc_auc = dict() for i in range(n_classes): fpr[i], tpr[i], _ = roc_curve(y_test[:, i], y_score[:, i]) roc_auc[i] = auc(fpr[i], tpr[i]) # Compute micro-average ROC curve and ROC area fpr[\"micro\"], tpr[\"micro\"], _ = roc_curve(y_test.ravel(), y_score.ravel()) roc_auc[\"micro\"] = auc(fpr[\"micro\"], tpr[\"micro\"]) plt.figure() lw = 2 plt.plot(fpr[2], tpr[2], color='darkorange', lw=lw, label='ROC curve (area = %0.2f)' % roc_auc[2]) plt.plot([0, 1], [0, 1], color='navy', lw=lw, linestyle='--') plt.xlim([0.0, 1.0]) plt.ylim([0.0, 1.05]) plt.xlabel('False Positive Rate') plt.ylabel('True Positive Rate') plt.title('Receiver operating characteristic example') plt.legend(loc=\"lower right\") plt.show()","title":"Example 6 -- ROC Curve with decision_function"},{"location":"user_guide/classifier/StackingClassifier/#api","text":"","title":"API"},{"location":"user_guide/cluster/Kmeans/","text":"Kmeans: k-means clustering A implementation of k-means clustering. from mlxtend.cluster import Kmeans Overview Clustering falls into the category of unsupervised learning, a subfield of machine learning where the ground truth labels are not available to us in real-world applications. In clustering, our goal is to group samples by similarity (in k-means: Euclidean distance). The k-means algorithms can be summarized as follows: Randomly pick k centroids from the sample points as initial cluster centers. Assign each sample to the nearest centroid \\mu(j), \\; j \\in {1,...,k} . Move the centroids to the center of the samples that were assigned to it. Repeat steps 2 and 3 until the cluster assignments do not change or a user-defined tolerance or a maximum number of iterations is reached. References MacQueen, J. B. (1967). Some Methods for classification and Analysis of Multivariate Observations . Proceedings of 5th Berkeley Symposium on Mathematical Statistics and Probability. University of California Press. pp. 281\u2013297. MR 0214227. Zbl 0214.46201. Retrieved 2009-04-07. Example 1 - Three Blobs Load some sample data: import matplotlib.pyplot as plt from mlxtend.data import three_blobs_data X, y = three_blobs_data() plt.scatter(X[:, 0], X[:, 1], c='white') plt.show() Compute the cluster centroids: from mlxtend.cluster import Kmeans km = Kmeans(k=3, max_iter=50, random_seed=1, print_progress=3) km.fit(X) print('Iterations until convergence:', km.iterations_) print('Final centroids:\\n', km.centroids_) Iteration: 2/50 | Elapsed: 00:00:00 | ETA: 00:00:00 Iterations until convergence: 2 Final centroids: [[-1.5947298 2.92236966] [ 2.06521743 0.96137409] [ 0.9329651 4.35420713]] Visualize the cluster memberships: y_clust = km.predict(X) plt.scatter(X[y_clust == 0, 0], X[y_clust == 0, 1], s=50, c='lightgreen', marker='s', label='cluster 1') plt.scatter(X[y_clust == 1,0], X[y_clust == 1,1], s=50, c='orange', marker='o', label='cluster 2') plt.scatter(X[y_clust == 2,0], X[y_clust == 2,1], s=50, c='lightblue', marker='v', label='cluster 3') plt.scatter(km.centroids_[:,0], km.centroids_[:,1], s=250, marker='*', c='red', label='centroids') plt.legend(loc='lower left', scatterpoints=1) plt.grid() plt.show() API Kmeans(k, max_iter=10, convergence_tolerance=1e-05, random_seed=None, print_progress=0) K-means clustering class. Added in 0.4.1dev Parameters k : int Number of clusters max_iter : int (default: 10) Number of iterations during cluster assignment. Cluster re-assignment stops automatically when the algorithm converged. convergence_tolerance : float (default: 1e-05) Compares current centroids with centroids of the previous iteration using the given tolerance (a small positive float)to determine if the algorithm converged early. random_seed : int (default: None) Set random state for the initial centroid assignment. print_progress : int (default: 0) Prints progress in fitting to stderr. 0: No output 1: Iterations elapsed 2: 1 plus time elapsed 3: 2 plus estimated time until completion Attributes centroids_ : 2d-array, shape={k, n_features} Feature values of the k cluster centroids. custers_ : dictionary The cluster assignments stored as a Python dictionary; the dictionary keys denote the cluster indeces and the items are Python lists of the sample indices that were assigned to each cluster. iterations_ : int Number of iterations until convergence. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/classifier/Kmeans/ Methods fit(X, init_params=True) Learn model from training data. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. init_params : bool (default: True) Re-initializes model parameters prior to fitting. Set False to continue training with weights from a previous model fitting. Returns self : object predict(X) Predict targets from X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns target_values : array-like, shape = [n_samples] Predicted target values.","title":"Kmeans: k-means clustering"},{"location":"user_guide/cluster/Kmeans/#kmeans-k-means-clustering","text":"A implementation of k-means clustering. from mlxtend.cluster import Kmeans","title":"Kmeans: k-means clustering"},{"location":"user_guide/cluster/Kmeans/#overview","text":"Clustering falls into the category of unsupervised learning, a subfield of machine learning where the ground truth labels are not available to us in real-world applications. In clustering, our goal is to group samples by similarity (in k-means: Euclidean distance). The k-means algorithms can be summarized as follows: Randomly pick k centroids from the sample points as initial cluster centers. Assign each sample to the nearest centroid \\mu(j), \\; j \\in {1,...,k} . Move the centroids to the center of the samples that were assigned to it. Repeat steps 2 and 3 until the cluster assignments do not change or a user-defined tolerance or a maximum number of iterations is reached.","title":"Overview"},{"location":"user_guide/cluster/Kmeans/#references","text":"MacQueen, J. B. (1967). Some Methods for classification and Analysis of Multivariate Observations . Proceedings of 5th Berkeley Symposium on Mathematical Statistics and Probability. University of California Press. pp. 281\u2013297. MR 0214227. Zbl 0214.46201. Retrieved 2009-04-07.","title":"References"},{"location":"user_guide/cluster/Kmeans/#example-1-three-blobs","text":"","title":"Example 1 - Three Blobs"},{"location":"user_guide/cluster/Kmeans/#load-some-sample-data","text":"import matplotlib.pyplot as plt from mlxtend.data import three_blobs_data X, y = three_blobs_data() plt.scatter(X[:, 0], X[:, 1], c='white') plt.show()","title":"Load some sample data:"},{"location":"user_guide/cluster/Kmeans/#compute-the-cluster-centroids","text":"from mlxtend.cluster import Kmeans km = Kmeans(k=3, max_iter=50, random_seed=1, print_progress=3) km.fit(X) print('Iterations until convergence:', km.iterations_) print('Final centroids:\\n', km.centroids_) Iteration: 2/50 | Elapsed: 00:00:00 | ETA: 00:00:00 Iterations until convergence: 2 Final centroids: [[-1.5947298 2.92236966] [ 2.06521743 0.96137409] [ 0.9329651 4.35420713]]","title":"Compute the cluster centroids:"},{"location":"user_guide/cluster/Kmeans/#visualize-the-cluster-memberships","text":"y_clust = km.predict(X) plt.scatter(X[y_clust == 0, 0], X[y_clust == 0, 1], s=50, c='lightgreen', marker='s', label='cluster 1') plt.scatter(X[y_clust == 1,0], X[y_clust == 1,1], s=50, c='orange', marker='o', label='cluster 2') plt.scatter(X[y_clust == 2,0], X[y_clust == 2,1], s=50, c='lightblue', marker='v', label='cluster 3') plt.scatter(km.centroids_[:,0], km.centroids_[:,1], s=250, marker='*', c='red', label='centroids') plt.legend(loc='lower left', scatterpoints=1) plt.grid() plt.show()","title":"Visualize the cluster memberships:"},{"location":"user_guide/cluster/Kmeans/#api","text":"Kmeans(k, max_iter=10, convergence_tolerance=1e-05, random_seed=None, print_progress=0) K-means clustering class. Added in 0.4.1dev Parameters k : int Number of clusters max_iter : int (default: 10) Number of iterations during cluster assignment. Cluster re-assignment stops automatically when the algorithm converged. convergence_tolerance : float (default: 1e-05) Compares current centroids with centroids of the previous iteration using the given tolerance (a small positive float)to determine if the algorithm converged early. random_seed : int (default: None) Set random state for the initial centroid assignment. print_progress : int (default: 0) Prints progress in fitting to stderr. 0: No output 1: Iterations elapsed 2: 1 plus time elapsed 3: 2 plus estimated time until completion Attributes centroids_ : 2d-array, shape={k, n_features} Feature values of the k cluster centroids. custers_ : dictionary The cluster assignments stored as a Python dictionary; the dictionary keys denote the cluster indeces and the items are Python lists of the sample indices that were assigned to each cluster. iterations_ : int Number of iterations until convergence. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/classifier/Kmeans/","title":"API"},{"location":"user_guide/cluster/Kmeans/#methods","text":"fit(X, init_params=True) Learn model from training data. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. init_params : bool (default: True) Re-initializes model parameters prior to fitting. Set False to continue training with weights from a previous model fitting. Returns self : object predict(X) Predict targets from X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns target_values : array-like, shape = [n_samples] Predicted target values.","title":"Methods"},{"location":"user_guide/data/autompg_data/","text":"autompg_data: The Auto-MPG dataset for regression A function that loads the autompg dataset into NumPy arrays. from mlxtend.data import autompg_data Overview The Auto-MPG dataset for regression analysis. The target ( y ) is defined as the miles per gallon (mpg) for 392 automobiles (6 rows containing \"NaN\"s have been removed. The 8 feature columns are: Features cylinders: multi-valued discrete displacement: continuous horsepower: continuous weight: continuous acceleration: continuous model year: multi-valued discrete origin: multi-valued discrete car name: string (unique for each instance) Number of samples: 392 Target variable (continuous): mpg References Source: https://archive.ics.uci.edu/ml/datasets/Auto+MPG Quinlan,R. (1993). Combining Instance-Based and Model-Based Learning. In Proceedings on the Tenth International Conference of Machine Learning, 236-243, University of Massachusetts, Amherst. Morgan Kaufmann. Example - Dataset overview from mlxtend.data import autompg_data X, y = autompg_data() print('Dimensions: %s x %s' % (X.shape[0], X.shape[1])) print('\\nHeader: %s' % ['cylinders', 'displacement', 'horsepower', 'weight', 'acceleration', 'model year', 'origin', 'car name']) print('1st row', X[0]) Dimensions: 392 x 8 Header: ['cylinders', 'displacement', 'horsepower', 'weight', 'acceleration', 'model year', 'origin', 'car name'] 1st row [ 8.00000000e+00 3.07000000e+02 1.30000000e+02 3.50400000e+03 1.20000000e+01 7.00000000e+01 1.00000000e+00 nan] Note that the feature array contains a str column (\"car name\"), thus it is recommended to pick the features as needed and convert it into a float array for further analysis. The example below shows how to get rid of the car name column and cast the NumPy array as a float array. X[:, :-1].astype(float) array([[ 8. , 307. , 130. , ..., 12. , 70. , 1. ], [ 8. , 350. , 165. , ..., 11.5, 70. , 1. ], [ 8. , 318. , 150. , ..., 11. , 70. , 1. ], ..., [ 4. , 135. , 84. , ..., 11.6, 82. , 1. ], [ 4. , 120. , 79. , ..., 18.6, 82. , 1. ], [ 4. , 119. , 82. , ..., 19.4, 82. , 1. ]]) API autompg_data() Auto MPG dataset. Source : https://archive.ics.uci.edu/ml/datasets/Auto+MPG Number of samples : 392 Continuous target variable : mpg Dataset Attributes: 1) cylinders: multi-valued discrete 2) displacement: continuous 3) horsepower: continuous 4) weight: continuous 5) acceleration: continuous 6) model year: multi-valued discrete 7) origin: multi-valued discrete 8) car name: string (unique for each instance) Returns X, y : [n_samples, n_features], [n_targets] X is the feature matrix with 392 auto samples as rows and 8 feature columns (6 rows with NaNs removed). y is a 1-dimensional array of the target MPG values. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/data/autompg_data/","title":"autompg_data: The Auto-MPG dataset for regression"},{"location":"user_guide/data/autompg_data/#autompg_data-the-auto-mpg-dataset-for-regression","text":"A function that loads the autompg dataset into NumPy arrays. from mlxtend.data import autompg_data","title":"autompg_data: The Auto-MPG dataset for regression"},{"location":"user_guide/data/autompg_data/#overview","text":"The Auto-MPG dataset for regression analysis. The target ( y ) is defined as the miles per gallon (mpg) for 392 automobiles (6 rows containing \"NaN\"s have been removed. The 8 feature columns are: Features cylinders: multi-valued discrete displacement: continuous horsepower: continuous weight: continuous acceleration: continuous model year: multi-valued discrete origin: multi-valued discrete car name: string (unique for each instance) Number of samples: 392 Target variable (continuous): mpg","title":"Overview"},{"location":"user_guide/data/autompg_data/#references","text":"Source: https://archive.ics.uci.edu/ml/datasets/Auto+MPG Quinlan,R. (1993). Combining Instance-Based and Model-Based Learning. In Proceedings on the Tenth International Conference of Machine Learning, 236-243, University of Massachusetts, Amherst. Morgan Kaufmann.","title":"References"},{"location":"user_guide/data/autompg_data/#example-dataset-overview","text":"from mlxtend.data import autompg_data X, y = autompg_data() print('Dimensions: %s x %s' % (X.shape[0], X.shape[1])) print('\\nHeader: %s' % ['cylinders', 'displacement', 'horsepower', 'weight', 'acceleration', 'model year', 'origin', 'car name']) print('1st row', X[0]) Dimensions: 392 x 8 Header: ['cylinders', 'displacement', 'horsepower', 'weight', 'acceleration', 'model year', 'origin', 'car name'] 1st row [ 8.00000000e+00 3.07000000e+02 1.30000000e+02 3.50400000e+03 1.20000000e+01 7.00000000e+01 1.00000000e+00 nan] Note that the feature array contains a str column (\"car name\"), thus it is recommended to pick the features as needed and convert it into a float array for further analysis. The example below shows how to get rid of the car name column and cast the NumPy array as a float array. X[:, :-1].astype(float) array([[ 8. , 307. , 130. , ..., 12. , 70. , 1. ], [ 8. , 350. , 165. , ..., 11.5, 70. , 1. ], [ 8. , 318. , 150. , ..., 11. , 70. , 1. ], ..., [ 4. , 135. , 84. , ..., 11.6, 82. , 1. ], [ 4. , 120. , 79. , ..., 18.6, 82. , 1. ], [ 4. , 119. , 82. , ..., 19.4, 82. , 1. ]])","title":"Example - Dataset overview"},{"location":"user_guide/data/autompg_data/#api","text":"autompg_data() Auto MPG dataset. Source : https://archive.ics.uci.edu/ml/datasets/Auto+MPG Number of samples : 392 Continuous target variable : mpg Dataset Attributes: 1) cylinders: multi-valued discrete 2) displacement: continuous 3) horsepower: continuous 4) weight: continuous 5) acceleration: continuous 6) model year: multi-valued discrete 7) origin: multi-valued discrete 8) car name: string (unique for each instance) Returns X, y : [n_samples, n_features], [n_targets] X is the feature matrix with 392 auto samples as rows and 8 feature columns (6 rows with NaNs removed). y is a 1-dimensional array of the target MPG values. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/data/autompg_data/","title":"API"},{"location":"user_guide/data/boston_housing_data/","text":"boston_housing_data: The Boston housing dataset for regression A function that loads the boston_housing_data dataset into NumPy arrays. from mlxtend.data import boston_housing_data Overview The Boston Housing dataset for regression analysis. Features CRIM: per capita crime rate by town ZN: proportion of residential land zoned for lots over 25,000 sq.ft. INDUS: proportion of non-retail business acres per town CHAS: Charles River dummy variable (= 1 if tract bounds river; 0 otherwise) NOX: nitric oxides concentration (parts per 10 million) RM: average number of rooms per dwelling AGE: proportion of owner-occupied units built prior to 1940 DIS: weighted distances to five Boston employment centres RAD: index of accessibility to radial highways TAX: full-value property-tax rate per $10,000 PTRATIO: pupil-teacher ratio by town B: 1000(Bk - 0.63)^2 where Bk is the proportion of b. by town LSTAT: % lower status of the population Number of samples: 506 Target variable (continuous): MEDV, Median value of owner-occupied homes in $1000's References Source: https://archive.ics.uci.edu/ml/datasets/Wine Harrison, D. and Rubinfeld, D.L. 'Hedonic prices and the demand for clean air', J. Environ. Economics & Management, vol.5, 81-102, 1978. Example 1 - Dataset overview from mlxtend.data import boston_housing_data X, y = boston_housing_data() print('Dimensions: %s x %s' % (X.shape[0], X.shape[1])) print('1st row', X[0]) (506, 14) Dimensions: 506 x 13 1st row [ 6.32000000e-03 1.80000000e+01 2.31000000e+00 0.00000000e+00 5.38000000e-01 6.57500000e+00 6.52000000e+01 4.09000000e+00 1.00000000e+00 2.96000000e+02 1.53000000e+01 3.96900000e+02 4.98000000e+00] API boston_housing_data() Boston Housing dataset. Source : https://archive.ics.uci.edu/ml/datasets/Housing Number of samples : 506 Continuous target variable : MEDV MEDV = Median value of owner-occupied homes in $1000's Dataset Attributes: 1) CRIM per capita crime rate by town 2) ZN proportion of residential land zoned for lots over 25,000 sq.ft. 3) INDUS proportion of non-retail business acres per town 4) CHAS Charles River dummy variable (= 1 if tract bounds river; 0 otherwise) 5) NOX nitric oxides concentration (parts per 10 million) 6) RM average number of rooms per dwelling 7) AGE proportion of owner-occupied units built prior to 1940 8) DIS weighted distances to five Boston employment centres 9) RAD index of accessibility to radial highways 10) TAX full-value property-tax rate per $10,000 11) PTRATIO pupil-teacher ratio by town 12) B 1000(Bk - 0.63)^2 where Bk is the prop. of b. by town 13) LSTAT % lower status of the population Returns X, y : [n_samples, n_features], [n_class_labels] X is the feature matrix with 506 housing samples as rows and 13 feature columns. y is a 1-dimensional array of the continuous target variable MEDV Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/data/boston_housing_data/","title":"boston_housing_data: The Boston housing dataset for regression"},{"location":"user_guide/data/boston_housing_data/#boston_housing_data-the-boston-housing-dataset-for-regression","text":"A function that loads the boston_housing_data dataset into NumPy arrays. from mlxtend.data import boston_housing_data","title":"boston_housing_data: The Boston housing dataset for regression"},{"location":"user_guide/data/boston_housing_data/#overview","text":"The Boston Housing dataset for regression analysis. Features CRIM: per capita crime rate by town ZN: proportion of residential land zoned for lots over 25,000 sq.ft. INDUS: proportion of non-retail business acres per town CHAS: Charles River dummy variable (= 1 if tract bounds river; 0 otherwise) NOX: nitric oxides concentration (parts per 10 million) RM: average number of rooms per dwelling AGE: proportion of owner-occupied units built prior to 1940 DIS: weighted distances to five Boston employment centres RAD: index of accessibility to radial highways TAX: full-value property-tax rate per $10,000 PTRATIO: pupil-teacher ratio by town B: 1000(Bk - 0.63)^2 where Bk is the proportion of b. by town LSTAT: % lower status of the population Number of samples: 506 Target variable (continuous): MEDV, Median value of owner-occupied homes in $1000's","title":"Overview"},{"location":"user_guide/data/boston_housing_data/#references","text":"Source: https://archive.ics.uci.edu/ml/datasets/Wine Harrison, D. and Rubinfeld, D.L. 'Hedonic prices and the demand for clean air', J. Environ. Economics & Management, vol.5, 81-102, 1978.","title":"References"},{"location":"user_guide/data/boston_housing_data/#example-1-dataset-overview","text":"from mlxtend.data import boston_housing_data X, y = boston_housing_data() print('Dimensions: %s x %s' % (X.shape[0], X.shape[1])) print('1st row', X[0]) (506, 14) Dimensions: 506 x 13 1st row [ 6.32000000e-03 1.80000000e+01 2.31000000e+00 0.00000000e+00 5.38000000e-01 6.57500000e+00 6.52000000e+01 4.09000000e+00 1.00000000e+00 2.96000000e+02 1.53000000e+01 3.96900000e+02 4.98000000e+00]","title":"Example 1 - Dataset overview"},{"location":"user_guide/data/boston_housing_data/#api","text":"boston_housing_data() Boston Housing dataset. Source : https://archive.ics.uci.edu/ml/datasets/Housing Number of samples : 506 Continuous target variable : MEDV MEDV = Median value of owner-occupied homes in $1000's Dataset Attributes: 1) CRIM per capita crime rate by town 2) ZN proportion of residential land zoned for lots over 25,000 sq.ft. 3) INDUS proportion of non-retail business acres per town 4) CHAS Charles River dummy variable (= 1 if tract bounds river; 0 otherwise) 5) NOX nitric oxides concentration (parts per 10 million) 6) RM average number of rooms per dwelling 7) AGE proportion of owner-occupied units built prior to 1940 8) DIS weighted distances to five Boston employment centres 9) RAD index of accessibility to radial highways 10) TAX full-value property-tax rate per $10,000 11) PTRATIO pupil-teacher ratio by town 12) B 1000(Bk - 0.63)^2 where Bk is the prop. of b. by town 13) LSTAT % lower status of the population Returns X, y : [n_samples, n_features], [n_class_labels] X is the feature matrix with 506 housing samples as rows and 13 feature columns. y is a 1-dimensional array of the continuous target variable MEDV Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/data/boston_housing_data/","title":"API"},{"location":"user_guide/data/iris_data/","text":"iris_data: The 3-class iris dataset for classification A function that loads the iris dataset into NumPy arrays. from mlxtend.data import iris_data Overview The Iris dataset for classification. Features Sepal length Sepal width Petal length Petal width Number of samples: 150 Target variable (discrete): {50x Setosa, 50x Versicolor, 50x Virginica} References Source: https://archive.ics.uci.edu/ml/datasets/Iris Bache, K. & Lichman, M. (2013). UCI Machine Learning Repository. Irvine, CA: University of California, School of Information and Computer Science. Example 1 - Dataset overview from mlxtend.data import iris_data X, y = iris_data() print('Dimensions: %s x %s' % (X.shape[0], X.shape[1])) print('\\nHeader: %s' % ['sepal length', 'sepal width', 'petal length', 'petal width']) print('1st row', X[0]) Dimensions: 150 x 4 Header: ['sepal length', 'sepal width', 'petal length', 'petal width'] 1st row [5.1 3.5 1.4 0.2] import numpy as np print('Classes: Setosa, Versicolor, Virginica') print(np.unique(y)) print('Class distribution: %s' % np.bincount(y)) Classes: Setosa, Versicolor, Virginica [0 1 2] Class distribution: [50 50 50] API iris_data(version='uci') Iris flower dataset. Source : https://archive.ics.uci.edu/ml/datasets/Iris Number of samples : 150 Class labels : {0, 1, 2}, distribution: [50, 50, 50] 0 = setosa, 1 = versicolor, 2 = virginica. Dataset Attributes: 1) sepal length [cm] 2) sepal width [cm] 3) petal length [cm] 4) petal width [cm] Parameters version : string, optional (default: 'uci'). Version to use {'uci', 'corrected'}. 'uci' loads the dataset as deposited on the UCI machine learning repository, and 'corrected' provides the version that is consistent with Fisher's original paper. See Note for details. Returns X, y : [n_samples, n_features], [n_class_labels] X is the feature matrix with 150 flower samples as rows, and 4 feature columns sepal length, sepal width, petal length, and petal width. y is a 1-dimensional array of the class labels {0, 1, 2} Note The Iris dataset (originally collected by Edgar Anderson) and available in UCI's machine learning repository is different from the Iris dataset described in the original paper by R.A. Fisher [1]). Precisely, there are two data points (row number 34 and 37) in UCI's Machine Learning repository are different from the origianlly published Iris dataset. Also, the original version of the Iris Dataset, which can be loaded via version='corrected' is the same as the one in R. [1] . A. Fisher (1936). \"The use of multiple measurements in taxonomic problems\". Annals of Eugenics. 7 (2): 179\u2013188 Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/data/iris_data/","title":"iris_data: The 3-class iris dataset for classification"},{"location":"user_guide/data/iris_data/#iris_data-the-3-class-iris-dataset-for-classification","text":"A function that loads the iris dataset into NumPy arrays. from mlxtend.data import iris_data","title":"iris_data: The 3-class iris dataset for classification"},{"location":"user_guide/data/iris_data/#overview","text":"The Iris dataset for classification. Features Sepal length Sepal width Petal length Petal width Number of samples: 150 Target variable (discrete): {50x Setosa, 50x Versicolor, 50x Virginica}","title":"Overview"},{"location":"user_guide/data/iris_data/#references","text":"Source: https://archive.ics.uci.edu/ml/datasets/Iris Bache, K. & Lichman, M. (2013). UCI Machine Learning Repository. Irvine, CA: University of California, School of Information and Computer Science.","title":"References"},{"location":"user_guide/data/iris_data/#example-1-dataset-overview","text":"from mlxtend.data import iris_data X, y = iris_data() print('Dimensions: %s x %s' % (X.shape[0], X.shape[1])) print('\\nHeader: %s' % ['sepal length', 'sepal width', 'petal length', 'petal width']) print('1st row', X[0]) Dimensions: 150 x 4 Header: ['sepal length', 'sepal width', 'petal length', 'petal width'] 1st row [5.1 3.5 1.4 0.2] import numpy as np print('Classes: Setosa, Versicolor, Virginica') print(np.unique(y)) print('Class distribution: %s' % np.bincount(y)) Classes: Setosa, Versicolor, Virginica [0 1 2] Class distribution: [50 50 50]","title":"Example 1 - Dataset overview"},{"location":"user_guide/data/iris_data/#api","text":"iris_data(version='uci') Iris flower dataset. Source : https://archive.ics.uci.edu/ml/datasets/Iris Number of samples : 150 Class labels : {0, 1, 2}, distribution: [50, 50, 50] 0 = setosa, 1 = versicolor, 2 = virginica. Dataset Attributes: 1) sepal length [cm] 2) sepal width [cm] 3) petal length [cm] 4) petal width [cm] Parameters version : string, optional (default: 'uci'). Version to use {'uci', 'corrected'}. 'uci' loads the dataset as deposited on the UCI machine learning repository, and 'corrected' provides the version that is consistent with Fisher's original paper. See Note for details. Returns X, y : [n_samples, n_features], [n_class_labels] X is the feature matrix with 150 flower samples as rows, and 4 feature columns sepal length, sepal width, petal length, and petal width. y is a 1-dimensional array of the class labels {0, 1, 2} Note The Iris dataset (originally collected by Edgar Anderson) and available in UCI's machine learning repository is different from the Iris dataset described in the original paper by R.A. Fisher [1]). Precisely, there are two data points (row number 34 and 37) in UCI's Machine Learning repository are different from the origianlly published Iris dataset. Also, the original version of the Iris Dataset, which can be loaded via version='corrected' is the same as the one in R. [1] . A. Fisher (1936). \"The use of multiple measurements in taxonomic problems\". Annals of Eugenics. 7 (2): 179\u2013188 Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/data/iris_data/","title":"API"},{"location":"user_guide/data/loadlocal_mnist/","text":"loadlocal_mnist: A function for loading MNIST from the original ubyte files A utility function that loads the MNIST dataset from byte-form into NumPy arrays. from mlxtend.data import loadlocal_mnist Overview The MNIST dataset was constructed from two datasets of the US National Institute of Standards and Technology (NIST). The training set consists of handwritten digits from 250 different people, 50 percent high school students, and 50 percent employees from the Census Bureau. Note that the test set contains handwritten digits from different people following the same split. The MNIST dataset is publicly available at https://yann.lecun.com/exdb/mnist/ and consists of the following four parts: - Training set images: train-images-idx3-ubyte.gz (9.9 MB, 47 MB unzipped, and 60,000 samples) - Training set labels: train-labels-idx1-ubyte.gz (29 KB, 60 KB unzipped, and 60,000 labels) - Test set images: t10k-images-idx3-ubyte.gz (1.6 MB, 7.8 MB, unzipped and 10,000 samples) - Test set labels: t10k-labels-idx1-ubyte.gz (5 KB, 10 KB unzipped, and 10,000 labels) Features Each feature vector (row in the feature matrix) consists of 784 pixels (intensities) -- unrolled from the original 28x28 pixels images. Number of samples: 50000 images Target variable (discrete): Uniformly distributed class labels 0-9 corresponding to the respective handwritten digit shown in the image. References Source: https://yann.lecun.com/exdb/mnist/ Y. LeCun and C. Cortes. Mnist handwritten digit database. AT&T Labs [Online]. Available: https://yann. lecun. com/exdb/mnist, 2010. Example 1 Part 1 - Downloading the MNIST dataset 1) Download the MNIST files from Y. LeCun's website https://yann.lecun.com/exdb/mnist/train-images-idx3-ubyte.gz https://yann.lecun.com/exdb/mnist/train-labels-idx1-ubyte.gz https://yann.lecun.com/exdb/mnist/t10k-images-idx3-ubyte.gz https://yann.lecun.com/exdb/mnist/t10k-labels-idx1-ubyte.gz for example, via curl -O https://yann.lecun.com/exdb/mnist/train-images-idx3-ubyte.gz 2) Unzip the downloaded gzip archives for example, via gunzip t*-ubyte.gz Example 1 Part 2 - Loading MNIST into NumPy Arrays from mlxtend.data import loadlocal_mnist import platform if not platform.system() == 'Windows': X, y = loadlocal_mnist( images_path='train-images-idx3-ubyte', labels_path='train-labels-idx1-ubyte') else: X, y = loadlocal_mnist( images_path='train-images.idx3-ubyte', labels_path='train-labels.idx1-ubyte') print('Dimensions: %s x %s' % (X.shape[0], X.shape[1])) print('\\n1st row', X[0]) Dimensions: 60000 x 784 1st row [ 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 3 18 18 18 126 136 175 26 166 255 247 127 0 0 0 0 0 0 0 0 0 0 0 0 30 36 94 154 170 253 253 253 253 253 225 172 253 242 195 64 0 0 0 0 0 0 0 0 0 0 0 49 238 253 253 253 253 253 253 253 253 251 93 82 82 56 39 0 0 0 0 0 0 0 0 0 0 0 0 18 219 253 253 253 253 253 198 182 247 241 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 80 156 107 253 253 205 11 0 43 154 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 14 1 154 253 90 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 139 253 190 2 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 11 190 253 70 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 35 241 225 160 108 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 81 240 253 253 119 25 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 45 186 253 253 150 27 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 16 93 252 253 187 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 249 253 249 64 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 46 130 183 253 253 207 2 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 39 148 229 253 253 253 250 182 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 24 114 221 253 253 253 253 201 78 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 23 66 213 253 253 253 253 198 81 2 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 18 171 219 253 253 253 253 195 80 9 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 55 172 226 253 253 253 253 244 133 11 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 136 253 253 253 212 135 132 16 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0] import numpy as np print('Digits: 0 1 2 3 4 5 6 7 8 9') print('labels: %s' % np.unique(y)) print('Class distribution: %s' % np.bincount(y)) Digits: 0 1 2 3 4 5 6 7 8 9 labels: [0 1 2 3 4 5 6 7 8 9] Class distribution: [5923 6742 5958 6131 5842 5421 5918 6265 5851 5949] Store as CSV Files np.savetxt(fname='images.csv', X=X, delimiter=',', fmt='%d') np.savetxt(fname='labels.csv', X=y, delimiter=',', fmt='%d') API loadlocal_mnist(images_path, labels_path) Read MNIST from ubyte files. Parameters images_path : str path to the test or train MNIST ubyte file labels_path : str path to the test or train MNIST class labels file Returns images : [n_samples, n_pixels] numpy.array Pixel values of the images. labels : [n_samples] numpy array Target class labels Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/data/loadlocal_mnist/","title":"loadlocal_mnist: A function for loading MNIST from the original ubyte files"},{"location":"user_guide/data/loadlocal_mnist/#loadlocal_mnist-a-function-for-loading-mnist-from-the-original-ubyte-files","text":"A utility function that loads the MNIST dataset from byte-form into NumPy arrays. from mlxtend.data import loadlocal_mnist","title":"loadlocal_mnist: A function for loading MNIST from the original ubyte files"},{"location":"user_guide/data/loadlocal_mnist/#overview","text":"The MNIST dataset was constructed from two datasets of the US National Institute of Standards and Technology (NIST). The training set consists of handwritten digits from 250 different people, 50 percent high school students, and 50 percent employees from the Census Bureau. Note that the test set contains handwritten digits from different people following the same split. The MNIST dataset is publicly available at https://yann.lecun.com/exdb/mnist/ and consists of the following four parts: - Training set images: train-images-idx3-ubyte.gz (9.9 MB, 47 MB unzipped, and 60,000 samples) - Training set labels: train-labels-idx1-ubyte.gz (29 KB, 60 KB unzipped, and 60,000 labels) - Test set images: t10k-images-idx3-ubyte.gz (1.6 MB, 7.8 MB, unzipped and 10,000 samples) - Test set labels: t10k-labels-idx1-ubyte.gz (5 KB, 10 KB unzipped, and 10,000 labels) Features Each feature vector (row in the feature matrix) consists of 784 pixels (intensities) -- unrolled from the original 28x28 pixels images. Number of samples: 50000 images Target variable (discrete): Uniformly distributed class labels 0-9 corresponding to the respective handwritten digit shown in the image.","title":"Overview"},{"location":"user_guide/data/loadlocal_mnist/#references","text":"Source: https://yann.lecun.com/exdb/mnist/ Y. LeCun and C. Cortes. Mnist handwritten digit database. AT&T Labs [Online]. Available: https://yann. lecun. com/exdb/mnist, 2010.","title":"References"},{"location":"user_guide/data/loadlocal_mnist/#example-1-part-1-downloading-the-mnist-dataset","text":"1) Download the MNIST files from Y. LeCun's website https://yann.lecun.com/exdb/mnist/train-images-idx3-ubyte.gz https://yann.lecun.com/exdb/mnist/train-labels-idx1-ubyte.gz https://yann.lecun.com/exdb/mnist/t10k-images-idx3-ubyte.gz https://yann.lecun.com/exdb/mnist/t10k-labels-idx1-ubyte.gz for example, via curl -O https://yann.lecun.com/exdb/mnist/train-images-idx3-ubyte.gz 2) Unzip the downloaded gzip archives for example, via gunzip t*-ubyte.gz","title":"Example 1 Part 1 - Downloading the MNIST dataset"},{"location":"user_guide/data/loadlocal_mnist/#example-1-part-2-loading-mnist-into-numpy-arrays","text":"from mlxtend.data import loadlocal_mnist import platform if not platform.system() == 'Windows': X, y = loadlocal_mnist( images_path='train-images-idx3-ubyte', labels_path='train-labels-idx1-ubyte') else: X, y = loadlocal_mnist( images_path='train-images.idx3-ubyte', labels_path='train-labels.idx1-ubyte') print('Dimensions: %s x %s' % (X.shape[0], X.shape[1])) print('\\n1st row', X[0]) Dimensions: 60000 x 784 1st row [ 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 3 18 18 18 126 136 175 26 166 255 247 127 0 0 0 0 0 0 0 0 0 0 0 0 30 36 94 154 170 253 253 253 253 253 225 172 253 242 195 64 0 0 0 0 0 0 0 0 0 0 0 49 238 253 253 253 253 253 253 253 253 251 93 82 82 56 39 0 0 0 0 0 0 0 0 0 0 0 0 18 219 253 253 253 253 253 198 182 247 241 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 80 156 107 253 253 205 11 0 43 154 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 14 1 154 253 90 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 139 253 190 2 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 11 190 253 70 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 35 241 225 160 108 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 81 240 253 253 119 25 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 45 186 253 253 150 27 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 16 93 252 253 187 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 249 253 249 64 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 46 130 183 253 253 207 2 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 39 148 229 253 253 253 250 182 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 24 114 221 253 253 253 253 201 78 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 23 66 213 253 253 253 253 198 81 2 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 18 171 219 253 253 253 253 195 80 9 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 55 172 226 253 253 253 253 244 133 11 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 136 253 253 253 212 135 132 16 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0] import numpy as np print('Digits: 0 1 2 3 4 5 6 7 8 9') print('labels: %s' % np.unique(y)) print('Class distribution: %s' % np.bincount(y)) Digits: 0 1 2 3 4 5 6 7 8 9 labels: [0 1 2 3 4 5 6 7 8 9] Class distribution: [5923 6742 5958 6131 5842 5421 5918 6265 5851 5949]","title":"Example 1 Part 2 - Loading MNIST into NumPy Arrays"},{"location":"user_guide/data/loadlocal_mnist/#store-as-csv-files","text":"np.savetxt(fname='images.csv', X=X, delimiter=',', fmt='%d') np.savetxt(fname='labels.csv', X=y, delimiter=',', fmt='%d')","title":"Store as CSV Files"},{"location":"user_guide/data/loadlocal_mnist/#api","text":"loadlocal_mnist(images_path, labels_path) Read MNIST from ubyte files. Parameters images_path : str path to the test or train MNIST ubyte file labels_path : str path to the test or train MNIST class labels file Returns images : [n_samples, n_pixels] numpy.array Pixel values of the images. labels : [n_samples] numpy array Target class labels Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/data/loadlocal_mnist/","title":"API"},{"location":"user_guide/data/make_multiplexer_dataset/","text":"make_multiplexer_dataset: A function for creating multiplexer data Function that creates a dataset generated by a n-bit Boolean multiplexer for evaluating supervised learning algorithms. from mlxtend.data import make_multiplexer_dataset Overview The make_multiplexer_dataset function creates a dataset generated by an n-bit Boolean multiplexer. Such dataset represents a dataset generated by a simple rule, based on the behavior of a electric multiplexer, yet presents a relatively challenging classification problem for supervised learning algorithm with interactions between features (epistasis) as it may be encountered in many real-world scenarios [1]. The following illustration depicts a 6-bit multiplexer that consists of 2 address bits and 4 register bits. The address bits converted to decimal representation point to a position in the register bit. For example, if the address bits are \"00\" (0 in decimal), the address bits point to the register bit at position 0. The value of the register position pointed to determines the class label. For example, if the register bit at position is 0, the class label is 0. Vice versa, if the register bit at position 0 is 1, the class label is 1. In the example above, the address bits \"10\" (2 in decimal) point to the 3rd register position (as we start counting from index 0), which has a bit value of 1. Hence, the class label is 1. Below are a few more examples: Address bits: [0, 1], register bits: [1, 0, 1, 1], class label: 0 Address bits: [0, 1], register bits: [1, 1, 1, 0], class label: 1 Address bits: [1, 0], register bits: [1, 0, 0, 1], class label: 0 Address bits: [1, 1], register bits: [1, 1, 1, 0], class label: 0 Address bits: [0, 1], register bits: [0, 1, 1, 0], class label: 1 Address bits: [0, 1], register bits: [1, 0, 0, 1], class label: 0 Address bits: [0, 1], register bits: [0, 1, 1, 1], class label: 1 Address bits: [0, 1], register bits: [0, 0, 0, 0], class label: 0 Address bits: [1, 0], register bits: [1, 0, 1, 1], class label: 1 Address bits: [0, 1], register bits: [1, 1, 1, 1], class label: 1 Note that in the implementation of the multiplexer function, if the number of address bits is set to 2, this results in a 6 bit multiplexer as two bit can have 2^2=4 different register positions (2 bit + 4 bit = 6 bit). However, if we choose 3 address bits instead, 2^3=8 positions would be covered, resulting in a 11 bit (3 bit + 8 bit = 11 bit) multiplexer, and so forth. References [1] Urbanowicz, R. J., & Browne, W. N. (2017). Introduction to Learning Classifier Systems . Springer. Example 1 -- 6-bit multiplexer This simple example illustrates how to create dataset from a 6-bit multiplexer import numpy as np from mlxtend.data import make_multiplexer_dataset X, y = make_multiplexer_dataset(address_bits=2, sample_size=10, positive_class_ratio=0.5, shuffle=False, random_seed=123) print('Features:\\n', X) print('\\nClass labels:\\n', y) Features: [[0 1 0 1 0 1] [1 0 0 0 1 1] [0 1 1 1 0 0] [0 1 1 1 0 0] [0 0 1 1 0 0] [0 1 0 0 0 0] [0 1 1 0 1 1] [1 0 1 0 0 0] [1 0 0 1 0 1] [1 0 1 0 0 1]] Class labels: [1 1 1 1 1 0 0 0 0 0] API make_multiplexer_dataset(address_bits=2, sample_size=100, positive_class_ratio=0.5, shuffle=False, random_seed=None) Function to create a binary n-bit multiplexer dataset. New in mlxtend v0.9 Parameters address_bits : int (default: 2) A positive integer that determines the number of address bits in the multiplexer, which in turn determine the n-bit capacity of the multiplexer and therefore the number of features. The number of features is determined by the number of address bits. For example, 2 address bits will result in a 6 bit multiplexer and consequently 6 features (2 + 2^2 = 6). If address_bits=3 , then this results in an 11-bit multiplexer as (2 + 2^3 = 11) with 11 features. sample_size : int (default: 100) The total number of samples generated. positive_class_ratio : float (default: 0.5) The fraction (a float between 0 and 1) of samples in the sample_size d dataset that have class label 1. If positive_class_ratio=0.5 (default), then the ratio of class 0 and class 1 samples is perfectly balanced. shuffle : Bool (default: False) Whether or not to shuffle the features and labels. If False (default), the samples are returned in sorted order starting with sample_size /2 samples with class label 0 and followed by sample_size /2 samples with class label 1. random_seed : int (default: None) Random seed used for generating the multiplexer samples and shuffling. Returns X, y : [n_samples, n_features], [n_class_labels] X is the feature matrix with the number of samples equal to sample_size . The number of features is determined by the number of address bits. For instance, 2 address bits will result in a 6 bit multiplexer and consequently 6 features (2 + 2^2 = 6). All features are binary (values in {0, 1}). y is a 1-dimensional array of class labels in {0, 1}. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/data/make_multiplexer_dataset","title":"make_multiplexer_dataset: A function for creating multiplexer data"},{"location":"user_guide/data/make_multiplexer_dataset/#make_multiplexer_dataset-a-function-for-creating-multiplexer-data","text":"Function that creates a dataset generated by a n-bit Boolean multiplexer for evaluating supervised learning algorithms. from mlxtend.data import make_multiplexer_dataset","title":"make_multiplexer_dataset: A function for creating multiplexer data"},{"location":"user_guide/data/make_multiplexer_dataset/#overview","text":"The make_multiplexer_dataset function creates a dataset generated by an n-bit Boolean multiplexer. Such dataset represents a dataset generated by a simple rule, based on the behavior of a electric multiplexer, yet presents a relatively challenging classification problem for supervised learning algorithm with interactions between features (epistasis) as it may be encountered in many real-world scenarios [1]. The following illustration depicts a 6-bit multiplexer that consists of 2 address bits and 4 register bits. The address bits converted to decimal representation point to a position in the register bit. For example, if the address bits are \"00\" (0 in decimal), the address bits point to the register bit at position 0. The value of the register position pointed to determines the class label. For example, if the register bit at position is 0, the class label is 0. Vice versa, if the register bit at position 0 is 1, the class label is 1. In the example above, the address bits \"10\" (2 in decimal) point to the 3rd register position (as we start counting from index 0), which has a bit value of 1. Hence, the class label is 1. Below are a few more examples: Address bits: [0, 1], register bits: [1, 0, 1, 1], class label: 0 Address bits: [0, 1], register bits: [1, 1, 1, 0], class label: 1 Address bits: [1, 0], register bits: [1, 0, 0, 1], class label: 0 Address bits: [1, 1], register bits: [1, 1, 1, 0], class label: 0 Address bits: [0, 1], register bits: [0, 1, 1, 0], class label: 1 Address bits: [0, 1], register bits: [1, 0, 0, 1], class label: 0 Address bits: [0, 1], register bits: [0, 1, 1, 1], class label: 1 Address bits: [0, 1], register bits: [0, 0, 0, 0], class label: 0 Address bits: [1, 0], register bits: [1, 0, 1, 1], class label: 1 Address bits: [0, 1], register bits: [1, 1, 1, 1], class label: 1 Note that in the implementation of the multiplexer function, if the number of address bits is set to 2, this results in a 6 bit multiplexer as two bit can have 2^2=4 different register positions (2 bit + 4 bit = 6 bit). However, if we choose 3 address bits instead, 2^3=8 positions would be covered, resulting in a 11 bit (3 bit + 8 bit = 11 bit) multiplexer, and so forth.","title":"Overview"},{"location":"user_guide/data/make_multiplexer_dataset/#references","text":"[1] Urbanowicz, R. J., & Browne, W. N. (2017). Introduction to Learning Classifier Systems . Springer.","title":"References"},{"location":"user_guide/data/make_multiplexer_dataset/#example-1-6-bit-multiplexer","text":"This simple example illustrates how to create dataset from a 6-bit multiplexer import numpy as np from mlxtend.data import make_multiplexer_dataset X, y = make_multiplexer_dataset(address_bits=2, sample_size=10, positive_class_ratio=0.5, shuffle=False, random_seed=123) print('Features:\\n', X) print('\\nClass labels:\\n', y) Features: [[0 1 0 1 0 1] [1 0 0 0 1 1] [0 1 1 1 0 0] [0 1 1 1 0 0] [0 0 1 1 0 0] [0 1 0 0 0 0] [0 1 1 0 1 1] [1 0 1 0 0 0] [1 0 0 1 0 1] [1 0 1 0 0 1]] Class labels: [1 1 1 1 1 0 0 0 0 0]","title":"Example 1 -- 6-bit multiplexer"},{"location":"user_guide/data/make_multiplexer_dataset/#api","text":"make_multiplexer_dataset(address_bits=2, sample_size=100, positive_class_ratio=0.5, shuffle=False, random_seed=None) Function to create a binary n-bit multiplexer dataset. New in mlxtend v0.9 Parameters address_bits : int (default: 2) A positive integer that determines the number of address bits in the multiplexer, which in turn determine the n-bit capacity of the multiplexer and therefore the number of features. The number of features is determined by the number of address bits. For example, 2 address bits will result in a 6 bit multiplexer and consequently 6 features (2 + 2^2 = 6). If address_bits=3 , then this results in an 11-bit multiplexer as (2 + 2^3 = 11) with 11 features. sample_size : int (default: 100) The total number of samples generated. positive_class_ratio : float (default: 0.5) The fraction (a float between 0 and 1) of samples in the sample_size d dataset that have class label 1. If positive_class_ratio=0.5 (default), then the ratio of class 0 and class 1 samples is perfectly balanced. shuffle : Bool (default: False) Whether or not to shuffle the features and labels. If False (default), the samples are returned in sorted order starting with sample_size /2 samples with class label 0 and followed by sample_size /2 samples with class label 1. random_seed : int (default: None) Random seed used for generating the multiplexer samples and shuffling. Returns X, y : [n_samples, n_features], [n_class_labels] X is the feature matrix with the number of samples equal to sample_size . The number of features is determined by the number of address bits. For instance, 2 address bits will result in a 6 bit multiplexer and consequently 6 features (2 + 2^2 = 6). All features are binary (values in {0, 1}). y is a 1-dimensional array of class labels in {0, 1}. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/data/make_multiplexer_dataset","title":"API"},{"location":"user_guide/data/mnist_data/","text":"mnist_data: A subset of the MNIST dataset for classification A function that loads the MNIST dataset into NumPy arrays. from mlxtend.data import mnist_data Overview The MNIST dataset was constructed from two datasets of the US National Institute of Standards and Technology (NIST). The training set consists of handwritten digits from 250 different people, 50 percent high school students, and 50 percent employees from the Census Bureau. Note that the test set contains handwritten digits from different people following the same split. Features Each feature vector (row in the feature matrix) consists of 784 pixels (intensities) -- unrolled from the original 28x28 pixels images. Number of samples: A subset of 5000 images (the first 500 digits of each class) Target variable (discrete): {500x 0, ..., 500x 9} References Source: https://yann.lecun.com/exdb/mnist/ Y. LeCun and C. Cortes. Mnist handwritten digit database. AT&T Labs [Online]. Available: https://yann.lecun.com/exdb/mnist , 2010. Example 1 - Dataset overview from mlxtend.data import mnist_data X, y = mnist_data() print('Dimensions: %s x %s' % (X.shape[0], X.shape[1])) print('1st row', X[0]) Dimensions: 5000 x 784 1st row [ 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 51. 159. 253. 159. 50. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 48. 238. 252. 252. 252. 237. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 54. 227. 253. 252. 239. 233. 252. 57. 6. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 10. 60. 224. 252. 253. 252. 202. 84. 252. 253. 122. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 163. 252. 252. 252. 253. 252. 252. 96. 189. 253. 167. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 51. 238. 253. 253. 190. 114. 253. 228. 47. 79. 255. 168. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 48. 238. 252. 252. 179. 12. 75. 121. 21. 0. 0. 253. 243. 50. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 38. 165. 253. 233. 208. 84. 0. 0. 0. 0. 0. 0. 253. 252. 165. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 7. 178. 252. 240. 71. 19. 28. 0. 0. 0. 0. 0. 0. 253. 252. 195. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 57. 252. 252. 63. 0. 0. 0. 0. 0. 0. 0. 0. 0. 253. 252. 195. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 198. 253. 190. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 255. 253. 196. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 76. 246. 252. 112. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 253. 252. 148. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 85. 252. 230. 25. 0. 0. 0. 0. 0. 0. 0. 0. 7. 135. 253. 186. 12. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 85. 252. 223. 0. 0. 0. 0. 0. 0. 0. 0. 7. 131. 252. 225. 71. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 85. 252. 145. 0. 0. 0. 0. 0. 0. 0. 48. 165. 252. 173. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 86. 253. 225. 0. 0. 0. 0. 0. 0. 114. 238. 253. 162. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 85. 252. 249. 146. 48. 29. 85. 178. 225. 253. 223. 167. 56. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 85. 252. 252. 252. 229. 215. 252. 252. 252. 196. 130. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 28. 199. 252. 252. 253. 252. 252. 233. 145. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 25. 128. 252. 253. 252. 141. 37. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.] import numpy as np print('Classes: Setosa, Versicolor, Virginica') print(np.unique(y)) print('Class distribution: %s' % np.bincount(y)) Classes: Setosa, Versicolor, Virginica [0 1 2 3 4 5 6 7 8 9] Class distribution: [500 500 500 500 500 500 500 500 500 500] Example 2 - Visualize MNIST %matplotlib inline import matplotlib.pyplot as plt def plot_digit(X, y, idx): img = X[idx].reshape(28,28) plt.imshow(img, cmap='Greys', interpolation='nearest') plt.title('true label: %d' % y[idx]) plt.show() plot_digit(X, y, 4) API mnist_data() 5000 samples from the MNIST handwritten digits dataset. Data Source : https://yann.lecun.com/exdb/mnist/ Returns X, y : [n_samples, n_features], [n_class_labels] X is the feature matrix with 5000 image samples as rows, each row consists of 28x28 pixels that were unrolled into 784 pixel feature vectors. y contains the 10 unique class labels 0-9. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/data/mnist_data/","title":"mnist_data: A subset of the MNIST dataset for classification"},{"location":"user_guide/data/mnist_data/#mnist_data-a-subset-of-the-mnist-dataset-for-classification","text":"A function that loads the MNIST dataset into NumPy arrays. from mlxtend.data import mnist_data","title":"mnist_data: A subset of the MNIST dataset for classification"},{"location":"user_guide/data/mnist_data/#overview","text":"The MNIST dataset was constructed from two datasets of the US National Institute of Standards and Technology (NIST). The training set consists of handwritten digits from 250 different people, 50 percent high school students, and 50 percent employees from the Census Bureau. Note that the test set contains handwritten digits from different people following the same split. Features Each feature vector (row in the feature matrix) consists of 784 pixels (intensities) -- unrolled from the original 28x28 pixels images. Number of samples: A subset of 5000 images (the first 500 digits of each class) Target variable (discrete): {500x 0, ..., 500x 9}","title":"Overview"},{"location":"user_guide/data/mnist_data/#references","text":"Source: https://yann.lecun.com/exdb/mnist/ Y. LeCun and C. Cortes. Mnist handwritten digit database. AT&T Labs [Online]. Available: https://yann.lecun.com/exdb/mnist , 2010.","title":"References"},{"location":"user_guide/data/mnist_data/#example-1-dataset-overview","text":"from mlxtend.data import mnist_data X, y = mnist_data() print('Dimensions: %s x %s' % (X.shape[0], X.shape[1])) print('1st row', X[0]) Dimensions: 5000 x 784 1st row [ 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 51. 159. 253. 159. 50. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 48. 238. 252. 252. 252. 237. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 54. 227. 253. 252. 239. 233. 252. 57. 6. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 10. 60. 224. 252. 253. 252. 202. 84. 252. 253. 122. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 163. 252. 252. 252. 253. 252. 252. 96. 189. 253. 167. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 51. 238. 253. 253. 190. 114. 253. 228. 47. 79. 255. 168. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 48. 238. 252. 252. 179. 12. 75. 121. 21. 0. 0. 253. 243. 50. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 38. 165. 253. 233. 208. 84. 0. 0. 0. 0. 0. 0. 253. 252. 165. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 7. 178. 252. 240. 71. 19. 28. 0. 0. 0. 0. 0. 0. 253. 252. 195. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 57. 252. 252. 63. 0. 0. 0. 0. 0. 0. 0. 0. 0. 253. 252. 195. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 198. 253. 190. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 255. 253. 196. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 76. 246. 252. 112. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 253. 252. 148. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 85. 252. 230. 25. 0. 0. 0. 0. 0. 0. 0. 0. 7. 135. 253. 186. 12. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 85. 252. 223. 0. 0. 0. 0. 0. 0. 0. 0. 7. 131. 252. 225. 71. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 85. 252. 145. 0. 0. 0. 0. 0. 0. 0. 48. 165. 252. 173. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 86. 253. 225. 0. 0. 0. 0. 0. 0. 114. 238. 253. 162. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 85. 252. 249. 146. 48. 29. 85. 178. 225. 253. 223. 167. 56. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 85. 252. 252. 252. 229. 215. 252. 252. 252. 196. 130. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 28. 199. 252. 252. 253. 252. 252. 233. 145. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 25. 128. 252. 253. 252. 141. 37. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.] import numpy as np print('Classes: Setosa, Versicolor, Virginica') print(np.unique(y)) print('Class distribution: %s' % np.bincount(y)) Classes: Setosa, Versicolor, Virginica [0 1 2 3 4 5 6 7 8 9] Class distribution: [500 500 500 500 500 500 500 500 500 500]","title":"Example 1 - Dataset overview"},{"location":"user_guide/data/mnist_data/#example-2-visualize-mnist","text":"%matplotlib inline import matplotlib.pyplot as plt def plot_digit(X, y, idx): img = X[idx].reshape(28,28) plt.imshow(img, cmap='Greys', interpolation='nearest') plt.title('true label: %d' % y[idx]) plt.show() plot_digit(X, y, 4)","title":"Example 2 - Visualize MNIST"},{"location":"user_guide/data/mnist_data/#api","text":"mnist_data() 5000 samples from the MNIST handwritten digits dataset. Data Source : https://yann.lecun.com/exdb/mnist/ Returns X, y : [n_samples, n_features], [n_class_labels] X is the feature matrix with 5000 image samples as rows, each row consists of 28x28 pixels that were unrolled into 784 pixel feature vectors. y contains the 10 unique class labels 0-9. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/data/mnist_data/","title":"API"},{"location":"user_guide/data/three_blobs_data/","text":"three_blobs_data: The synthetic blobs for classification A function that loads the three_blobs dataset into NumPy arrays. from mlxtend.data import three_blobs_data Overview A random dataset of 3 2D blobs for clustering. Number of samples : 150 Suggested labels \\in {0, 1, 2}, distribution: [50, 50, 50] References Example 1 - Dataset overview from mlxtend.data import three_blobs_data X, y = three_blobs_data() print('Dimensions: %s x %s' % (X.shape[0], X.shape[1])) print('1st row', X[0]) Dimensions: 150 x 2 1st row [ 2.60509732 1.22529553] import numpy as np print('Suggested cluster labels') print(np.unique(y)) print('Label distribution: %s' % np.bincount(y)) Suggested cluster labels [0 1 2] Label distribution: [50 50 50] import matplotlib.pyplot as plt plt.scatter(X[:,0], X[:,1], c='white', marker='o', s=50) plt.grid() plt.show() plt.scatter(X[y == 0, 0], X[y == 0, 1], s=50, c='lightgreen', marker='s', label='cluster 1') plt.scatter(X[y == 1,0], X[y == 1,1], s=50, c='orange', marker='o', label='cluster 2') plt.scatter(X[y == 2,0], X[y == 2,1], s=50, c='lightblue', marker='v', label='cluster 3') plt.legend(loc='lower left') plt.grid() plt.show() API three_blobs_data() A random dataset of 3 2D blobs for clustering. Number of samples : 150 Suggested labels : {0, 1, 2}, distribution: [50, 50, 50] Returns X, y : [n_samples, n_features], [n_cluster_labels] X is the feature matrix with 159 samples as rows and 2 feature columns. y is a 1-dimensional array of the 3 suggested cluster labels 0, 1, 2 Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/data/three_blobs_data","title":"three_blobs_data: The synthetic blobs for classification"},{"location":"user_guide/data/three_blobs_data/#three_blobs_data-the-synthetic-blobs-for-classification","text":"A function that loads the three_blobs dataset into NumPy arrays. from mlxtend.data import three_blobs_data","title":"three_blobs_data: The synthetic blobs for classification"},{"location":"user_guide/data/three_blobs_data/#overview","text":"A random dataset of 3 2D blobs for clustering. Number of samples : 150 Suggested labels \\in {0, 1, 2}, distribution: [50, 50, 50]","title":"Overview"},{"location":"user_guide/data/three_blobs_data/#references","text":"","title":"References"},{"location":"user_guide/data/three_blobs_data/#example-1-dataset-overview","text":"from mlxtend.data import three_blobs_data X, y = three_blobs_data() print('Dimensions: %s x %s' % (X.shape[0], X.shape[1])) print('1st row', X[0]) Dimensions: 150 x 2 1st row [ 2.60509732 1.22529553] import numpy as np print('Suggested cluster labels') print(np.unique(y)) print('Label distribution: %s' % np.bincount(y)) Suggested cluster labels [0 1 2] Label distribution: [50 50 50] import matplotlib.pyplot as plt plt.scatter(X[:,0], X[:,1], c='white', marker='o', s=50) plt.grid() plt.show() plt.scatter(X[y == 0, 0], X[y == 0, 1], s=50, c='lightgreen', marker='s', label='cluster 1') plt.scatter(X[y == 1,0], X[y == 1,1], s=50, c='orange', marker='o', label='cluster 2') plt.scatter(X[y == 2,0], X[y == 2,1], s=50, c='lightblue', marker='v', label='cluster 3') plt.legend(loc='lower left') plt.grid() plt.show()","title":"Example 1 - Dataset overview"},{"location":"user_guide/data/three_blobs_data/#api","text":"three_blobs_data() A random dataset of 3 2D blobs for clustering. Number of samples : 150 Suggested labels : {0, 1, 2}, distribution: [50, 50, 50] Returns X, y : [n_samples, n_features], [n_cluster_labels] X is the feature matrix with 159 samples as rows and 2 feature columns. y is a 1-dimensional array of the 3 suggested cluster labels 0, 1, 2 Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/data/three_blobs_data","title":"API"},{"location":"user_guide/data/wine_data/","text":"wine_data: A 3-class wine dataset for classification A function that loads the Wine dataset into NumPy arrays. from mlxtend.data import wine_data Overview The Wine dataset for classification. Samples 178 Features 13 Classes 3 Data Set Characteristics: Multivariate Attribute Characteristics: Integer, Real Associated Tasks: Classification Missing Values None column attribute 1) Class Label 2) Alcohol 3) Malic acid 4) Ash 5) Alcalinity of ash 6) Magnesium 7) Total phenols 8) Flavanoids 9) Nonflavanoid phenols 10) Proanthocyanins 11) Color intensity 12) Hue 13) OD280/OD315 of diluted wines 14) Proline class samples 0 59 1 71 2 48 References Forina, M. et al, PARVUS - An Extendible Package for Data Exploration, Classification and Correlation. Institute of Pharmaceutical and Food Analysis and Technologies, Via Brigata Salerno, 16147 Genoa, Italy. Source: https://archive.ics.uci.edu/ml/datasets/Wine Bache, K. & Lichman, M. (2013). UCI Machine Learning Repository. Irvine, CA: University of California, School of Information and Computer Science. Example 1 - Dataset overview from mlxtend.data import wine_data X, y = wine_data() print('Dimensions: %s x %s' % (X.shape[0], X.shape[1])) print('\\nHeader: %s' % ['alcohol', 'malic acid', 'ash', 'ash alcalinity', 'magnesium', 'total phenols', 'flavanoids', 'nonflavanoid phenols', 'proanthocyanins', 'color intensity', 'hue', 'OD280/OD315 of diluted wines', 'proline']) print('1st row', X[0]) Dimensions: 178 x 13 Header: ['alcohol', 'malic acid', 'ash', 'ash alcalinity', 'magnesium', 'total phenols', 'flavanoids', 'nonflavanoid phenols', 'proanthocyanins', 'color intensity', 'hue', 'OD280/OD315 of diluted wines', 'proline'] 1st row [ 1.42300000e+01 1.71000000e+00 2.43000000e+00 1.56000000e+01 1.27000000e+02 2.80000000e+00 3.06000000e+00 2.80000000e-01 2.29000000e+00 5.64000000e+00 1.04000000e+00 3.92000000e+00 1.06500000e+03] import numpy as np print('Classes: %s' % np.unique(y)) print('Class distribution: %s' % np.bincount(y)) Classes: [0 1 2] Class distribution: [59 71 48] API wine_data() Wine dataset. Source : https://archive.ics.uci.edu/ml/datasets/Wine Number of samples : 178 Class labels : {0, 1, 2}, distribution: [59, 71, 48] Dataset Attributes: 1) Alcohol 2) Malic acid 3) Ash 4) Alcalinity of ash 5) Magnesium 6) Total phenols 7) Flavanoids 8) Nonflavanoid phenols 9) Proanthocyanins 10) Color intensity 11) Hue 12) OD280/OD315 of diluted wines 13) Proline Returns X, y : [n_samples, n_features], [n_class_labels] X is the feature matrix with 178 wine samples as rows and 13 feature columns. y is a 1-dimensional array of the 3 class labels 0, 1, 2 Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/data/wine_data","title":"wine_data: A 3-class wine dataset for classification"},{"location":"user_guide/data/wine_data/#wine_data-a-3-class-wine-dataset-for-classification","text":"A function that loads the Wine dataset into NumPy arrays. from mlxtend.data import wine_data","title":"wine_data: A 3-class wine dataset for classification"},{"location":"user_guide/data/wine_data/#overview","text":"The Wine dataset for classification. Samples 178 Features 13 Classes 3 Data Set Characteristics: Multivariate Attribute Characteristics: Integer, Real Associated Tasks: Classification Missing Values None column attribute 1) Class Label 2) Alcohol 3) Malic acid 4) Ash 5) Alcalinity of ash 6) Magnesium 7) Total phenols 8) Flavanoids 9) Nonflavanoid phenols 10) Proanthocyanins 11) Color intensity 12) Hue 13) OD280/OD315 of diluted wines 14) Proline class samples 0 59 1 71 2 48","title":"Overview"},{"location":"user_guide/data/wine_data/#references","text":"Forina, M. et al, PARVUS - An Extendible Package for Data Exploration, Classification and Correlation. Institute of Pharmaceutical and Food Analysis and Technologies, Via Brigata Salerno, 16147 Genoa, Italy. Source: https://archive.ics.uci.edu/ml/datasets/Wine Bache, K. & Lichman, M. (2013). UCI Machine Learning Repository. Irvine, CA: University of California, School of Information and Computer Science.","title":"References"},{"location":"user_guide/data/wine_data/#example-1-dataset-overview","text":"from mlxtend.data import wine_data X, y = wine_data() print('Dimensions: %s x %s' % (X.shape[0], X.shape[1])) print('\\nHeader: %s' % ['alcohol', 'malic acid', 'ash', 'ash alcalinity', 'magnesium', 'total phenols', 'flavanoids', 'nonflavanoid phenols', 'proanthocyanins', 'color intensity', 'hue', 'OD280/OD315 of diluted wines', 'proline']) print('1st row', X[0]) Dimensions: 178 x 13 Header: ['alcohol', 'malic acid', 'ash', 'ash alcalinity', 'magnesium', 'total phenols', 'flavanoids', 'nonflavanoid phenols', 'proanthocyanins', 'color intensity', 'hue', 'OD280/OD315 of diluted wines', 'proline'] 1st row [ 1.42300000e+01 1.71000000e+00 2.43000000e+00 1.56000000e+01 1.27000000e+02 2.80000000e+00 3.06000000e+00 2.80000000e-01 2.29000000e+00 5.64000000e+00 1.04000000e+00 3.92000000e+00 1.06500000e+03] import numpy as np print('Classes: %s' % np.unique(y)) print('Class distribution: %s' % np.bincount(y)) Classes: [0 1 2] Class distribution: [59 71 48]","title":"Example 1 - Dataset overview"},{"location":"user_guide/data/wine_data/#api","text":"wine_data() Wine dataset. Source : https://archive.ics.uci.edu/ml/datasets/Wine Number of samples : 178 Class labels : {0, 1, 2}, distribution: [59, 71, 48] Dataset Attributes: 1) Alcohol 2) Malic acid 3) Ash 4) Alcalinity of ash 5) Magnesium 6) Total phenols 7) Flavanoids 8) Nonflavanoid phenols 9) Proanthocyanins 10) Color intensity 11) Hue 12) OD280/OD315 of diluted wines 13) Proline Returns X, y : [n_samples, n_features], [n_class_labels] X is the feature matrix with 178 wine samples as rows and 13 feature columns. y is a 1-dimensional array of the 3 class labels 0, 1, 2 Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/data/wine_data","title":"API"},{"location":"user_guide/evaluate/BootstrapOutOfBag/","text":"BootstrapOutOfBag: A scikit-learn compatible version of the out-of-bag bootstrap An implementation of the out-of-bag bootstrap to evaluate supervised learning algorithms. from mlxtend.evaluate import BootstrapOutOfBag Overview Originally, the bootstrap method aims to determine the statistical properties of an estimator when the underlying distribution was unknown and additional samples are not available. Now, in order to exploit this method for the evaluation of predictive models, such as hypotheses for classification and regression, we may prefer a slightly different approach to bootstrapping using the so-called Out-Of-Bag (OOB) or Leave-One-Out Bootstrap (LOOB) technique. Here, we use out-of-bag samples as test sets for evaluation instead of evaluating the model on the training data. Out-of-bag samples are the unique sets of instances that are not used for model fitting as shown in the figure below [1]. The figure above illustrates how three random bootstrap samples drawn from an exemplary ten-sample dataset ( X_1,X_2, ..., X_{10} ) and their out-of-bag sample for testing may look like. In practice, Bradley Efron and Robert Tibshirani recommend drawing 50 to 200 bootstrap samples as being sufficient for reliable estimates [2]. References [1] https://sebastianraschka.com/blog/2016/model-evaluation-selection-part2.html [2] Efron, Bradley, and Robert J. Tibshirani. An introduction to the bootstrap. CRC press, 1994. Management of Data (ACM SIGMOD '97), pages 265-276, 1997. Example 1 -- Evaluating the predictive performance of a model The BootstrapOutOfBag class mimics the behavior of scikit-learn's cross-validation classes, e.g., KFold : from mlxtend.evaluate import BootstrapOutOfBag import numpy as np oob = BootstrapOutOfBag(n_splits=3) for train, test in oob.split(np.array([1, 2, 3, 4, 5])): print(train, test) [4 2 1 3 3] [0] [2 4 1 2 1] [0 3] [4 3 3 4 1] [0 2] Consequently, we can use BootstrapOutOfBag objects via the cross_val_score method: from sklearn.datasets import load_iris from sklearn.linear_model import LogisticRegression from sklearn.model_selection import cross_val_score iris = load_iris() X = iris.data y = iris.target lr = LogisticRegression() print(cross_val_score(lr, X, y)) [ 0.96078431 0.92156863 0.95833333] print(cross_val_score(lr, X, y, cv=BootstrapOutOfBag(n_splits=3, random_seed=456))) [ 0.92727273 0.96226415 0.94444444] In practice, it is recommended to run at least 200 iterations, though: print('Mean accuracy: %.1f%%' % np.mean(100*cross_val_score( lr, X, y, cv=BootstrapOutOfBag(n_splits=200, random_seed=456)))) Mean accuracy: 94.8% Using the bootstrap, we can use the percentile method to compute the confidence bounds of the performance estimate. We pick our lower and upper confidence bounds as follows: ACC_{lower} = \\alpha_1th percentile of the ACC_{boot} distribution ACC_{lower} = \\alpha_2th percentile of the ACC_{boot} distribution where \\alpha_1 = \\alpha and \\alpha_2 = 1-\\alpha , and the degree of confidence to compute the 100 \\times (1-2 \\times \\alpha) confidence interval. For instance, to compute a 95% confidence interval, we pick \\alpha=0.025 to obtain the 2.5th and 97.5th percentiles of the b bootstrap samples distribution as the upper and lower confidence bounds. import matplotlib.pyplot as plt %matplotlib inline accuracies = cross_val_score(lr, X, y, cv=BootstrapOutOfBag(n_splits=1000, random_seed=456)) mean = np.mean(accuracies) lower = np.percentile(accuracies, 2.5) upper = np.percentile(accuracies, 97.5) fig, ax = plt.subplots(figsize=(8, 4)) ax.vlines(mean, [0], 40, lw=2.5, linestyle='-', label='mean') ax.vlines(lower, [0], 15, lw=2.5, linestyle='-.', label='CI95 percentile') ax.vlines(upper, [0], 15, lw=2.5, linestyle='-.') ax.hist(accuracies, bins=11, color='#0080ff', edgecolor=\"none\", alpha=0.3) plt.legend(loc='upper left') plt.show() API BootstrapOutOfBag(n_splits=200, random_seed=None) Parameters n_splits : int (default=200) Number of bootstrap iterations. Must be larger than 1. random_seed : int (default=None) If int, random_seed is the seed used by the random number generator. Returns train_idx : ndarray The training set indices for that split. test_idx : ndarray The testing set indices for that split. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/BootstrapOutOfBag/ Methods get_n_splits(X=None, y=None, groups=None) Returns the number of splitting iterations in the cross-validator Parameters X : object Always ignored, exists for compatibility with scikit-learn. y : object Always ignored, exists for compatibility with scikit-learn. groups : object Always ignored, exists for compatibility with scikit-learn. Returns n_splits : int Returns the number of splitting iterations in the cross-validator. split(X, y=None, groups=None) y : array-like or None (default: None) Argument is not used and only included as parameter for compatibility, similar to KFold in scikit-learn. groups : array-like or None (default: None) Argument is not used and only included as parameter for compatibility, similar to KFold in scikit-learn.","title":"BootstrapOutOfBag: A scikit-learn compatible version of the out-of-bag bootstrap"},{"location":"user_guide/evaluate/BootstrapOutOfBag/#bootstrapoutofbag-a-scikit-learn-compatible-version-of-the-out-of-bag-bootstrap","text":"An implementation of the out-of-bag bootstrap to evaluate supervised learning algorithms. from mlxtend.evaluate import BootstrapOutOfBag","title":"BootstrapOutOfBag: A scikit-learn compatible version of the out-of-bag bootstrap"},{"location":"user_guide/evaluate/BootstrapOutOfBag/#overview","text":"Originally, the bootstrap method aims to determine the statistical properties of an estimator when the underlying distribution was unknown and additional samples are not available. Now, in order to exploit this method for the evaluation of predictive models, such as hypotheses for classification and regression, we may prefer a slightly different approach to bootstrapping using the so-called Out-Of-Bag (OOB) or Leave-One-Out Bootstrap (LOOB) technique. Here, we use out-of-bag samples as test sets for evaluation instead of evaluating the model on the training data. Out-of-bag samples are the unique sets of instances that are not used for model fitting as shown in the figure below [1]. The figure above illustrates how three random bootstrap samples drawn from an exemplary ten-sample dataset ( X_1,X_2, ..., X_{10} ) and their out-of-bag sample for testing may look like. In practice, Bradley Efron and Robert Tibshirani recommend drawing 50 to 200 bootstrap samples as being sufficient for reliable estimates [2].","title":"Overview"},{"location":"user_guide/evaluate/BootstrapOutOfBag/#references","text":"[1] https://sebastianraschka.com/blog/2016/model-evaluation-selection-part2.html [2] Efron, Bradley, and Robert J. Tibshirani. An introduction to the bootstrap. CRC press, 1994. Management of Data (ACM SIGMOD '97), pages 265-276, 1997.","title":"References"},{"location":"user_guide/evaluate/BootstrapOutOfBag/#example-1-evaluating-the-predictive-performance-of-a-model","text":"The BootstrapOutOfBag class mimics the behavior of scikit-learn's cross-validation classes, e.g., KFold : from mlxtend.evaluate import BootstrapOutOfBag import numpy as np oob = BootstrapOutOfBag(n_splits=3) for train, test in oob.split(np.array([1, 2, 3, 4, 5])): print(train, test) [4 2 1 3 3] [0] [2 4 1 2 1] [0 3] [4 3 3 4 1] [0 2] Consequently, we can use BootstrapOutOfBag objects via the cross_val_score method: from sklearn.datasets import load_iris from sklearn.linear_model import LogisticRegression from sklearn.model_selection import cross_val_score iris = load_iris() X = iris.data y = iris.target lr = LogisticRegression() print(cross_val_score(lr, X, y)) [ 0.96078431 0.92156863 0.95833333] print(cross_val_score(lr, X, y, cv=BootstrapOutOfBag(n_splits=3, random_seed=456))) [ 0.92727273 0.96226415 0.94444444] In practice, it is recommended to run at least 200 iterations, though: print('Mean accuracy: %.1f%%' % np.mean(100*cross_val_score( lr, X, y, cv=BootstrapOutOfBag(n_splits=200, random_seed=456)))) Mean accuracy: 94.8% Using the bootstrap, we can use the percentile method to compute the confidence bounds of the performance estimate. We pick our lower and upper confidence bounds as follows: ACC_{lower} = \\alpha_1th percentile of the ACC_{boot} distribution ACC_{lower} = \\alpha_2th percentile of the ACC_{boot} distribution where \\alpha_1 = \\alpha and \\alpha_2 = 1-\\alpha , and the degree of confidence to compute the 100 \\times (1-2 \\times \\alpha) confidence interval. For instance, to compute a 95% confidence interval, we pick \\alpha=0.025 to obtain the 2.5th and 97.5th percentiles of the b bootstrap samples distribution as the upper and lower confidence bounds. import matplotlib.pyplot as plt %matplotlib inline accuracies = cross_val_score(lr, X, y, cv=BootstrapOutOfBag(n_splits=1000, random_seed=456)) mean = np.mean(accuracies) lower = np.percentile(accuracies, 2.5) upper = np.percentile(accuracies, 97.5) fig, ax = plt.subplots(figsize=(8, 4)) ax.vlines(mean, [0], 40, lw=2.5, linestyle='-', label='mean') ax.vlines(lower, [0], 15, lw=2.5, linestyle='-.', label='CI95 percentile') ax.vlines(upper, [0], 15, lw=2.5, linestyle='-.') ax.hist(accuracies, bins=11, color='#0080ff', edgecolor=\"none\", alpha=0.3) plt.legend(loc='upper left') plt.show()","title":"Example 1 -- Evaluating the predictive performance of a model"},{"location":"user_guide/evaluate/BootstrapOutOfBag/#api","text":"BootstrapOutOfBag(n_splits=200, random_seed=None) Parameters n_splits : int (default=200) Number of bootstrap iterations. Must be larger than 1. random_seed : int (default=None) If int, random_seed is the seed used by the random number generator. Returns train_idx : ndarray The training set indices for that split. test_idx : ndarray The testing set indices for that split. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/BootstrapOutOfBag/","title":"API"},{"location":"user_guide/evaluate/BootstrapOutOfBag/#methods","text":"get_n_splits(X=None, y=None, groups=None) Returns the number of splitting iterations in the cross-validator Parameters X : object Always ignored, exists for compatibility with scikit-learn. y : object Always ignored, exists for compatibility with scikit-learn. groups : object Always ignored, exists for compatibility with scikit-learn. Returns n_splits : int Returns the number of splitting iterations in the cross-validator. split(X, y=None, groups=None) y : array-like or None (default: None) Argument is not used and only included as parameter for compatibility, similar to KFold in scikit-learn. groups : array-like or None (default: None) Argument is not used and only included as parameter for compatibility, similar to KFold in scikit-learn.","title":"Methods"},{"location":"user_guide/evaluate/GroupTimeSeriesSplit/","text":"GroupTimeSeriesSplit: A scikit-learn compatible version of the time series validation with groups A scikit-learn-compatible time series cross-validator that supports non-overlapping groups. from mlxtend.evaluate import GroupTimeSeriesSplit Overview Time series tasks in machine learning require special type of validation, because the time order of the objects is important for a fairer evaluation of an ML model\u2019s quality. Also there can be different time units for splitting the data for different tasks - hours, days, months etc. Here, we use time series validation with support of the groups which can be flexibly configured along with other parameters: Test size Train size Number of splits Gap size Shift size Window type This GroupTimeSeriesSplit implementation is inspired by scikit-learn's TimeSeriesSplit but it has several advantages: GroupTimeSeriesSplit lets you group data as you want before splitting, while TimeSeriesSplit only considers the record level. It can be used for both holdout validation (n_splits=1) and cross-validation (n_splits>=2), whereas TimeSeriesSplit can be used only for the latter case. TimeSeriesSplit uses only an expanding window, while for this implementation you can choose between both rolling and expanding window types. GroupTimeSeriesSplit offers additional control for splitting using an additional shift size parameter. There are several features that need to be taken into account: GroupTimeSeriesSplit is compatible with sklearn-learn API. Numbers or custom non-numeric values can be used as groups However, groups should be consecutive Specifying the test size with either a) the train size or b) the number of splits is required parameters for splitting If full data can\u2019t be used with specific parameters, the most recent data is considered for splitting If splitting is impossible (e.g., there is not enough data to split) using specified parameters, an exception will be raised Before we illustrate the usage of GroupTimeSeriesSplit in the following examples below, let's set up a DummyClassifier that we will reuse in the following sections. Also, let's import the libraries we will be using in the following examples: import numpy as np import pandas as pd from sklearn.dummy import DummyClassifier from sklearn.model_selection import cross_val_score from mlxtend.evaluate.time_series import ( GroupTimeSeriesSplit, plot_splits, print_cv_info, print_split_info, ) Prepare sample data For the following examples, we are creating sample dataset consisting of 16 training data points with corresponding targets. Features and targets Let's assume that we have one numeric feature and target for the binary classification task. data = [[0], [7], [6], [4], [4], [8], [0], [6], [2], [0], [5], [9], [7], [7], [7], [7]] target = [1, 0, 1, 0, 1, 0, 0, 1, 1, 1, 0, 1, 1, 0, 0, 0] X = pd.DataFrame(data, columns=[\"num_feature\"]) y = pd.Series(target, name=\"target\") Group numbers We create 6 different groups so that the first training example belongs to group 0, the next 4 to group 1, and so forth. These groups do not have to be in ascending order (as in this dataset), but they must be consecutive. groups = np.array([0, 1, 1, 1, 1, 2, 2, 2, 3, 3, 4, 4, 5, 5, 5, 5]) groups array([0, 1, 1, 1, 1, 2, 2, 2, 3, 3, 4, 4, 5, 5, 5, 5]) Note that the following i another example of a correct group ordering (not sorted but consecutive): np.array([5, 5, 5, 5, 1, 1, 1, 1, 3, 3, 2, 2, 2, 4, 4, 0]) However, the example below shows an incorrect group ordering (not consecutive), which is not compatible with GroupTimeSeriesSplit : np.array([0, 1, 1, 1, 1, 2, 2, 2, 3, 3, 4, 4, 2, 2, 2, 2]) Group names (months) We will add months as the index according to the specified groups for a more illustrative example. months_map = {i: f\"2021-0{i+1}\" for i in range(6)} months = np.array([months_map[group] for group in groups]) months array(['2021-01', '2021-02', '2021-02', '2021-02', '2021-02', '2021-03', '2021-03', '2021-03', '2021-04', '2021-04', '2021-05', '2021-05', '2021-06', '2021-06', '2021-06', '2021-06'], dtype='<U7') X = X.set_index(months) Example 1 -- Multiple training groups (with train size specified) Let's create a time series splitting with a training dataset that consists of 3 groups. And we will use 1 group for testing. In this case, the number of splits will be calculated automatically since both the training set and the test set sizes are specified. The first 3 rows in the visualization describe how each split is distributed among groups. The groups are visualized in the last row where each color represents different group. cv_args = {\"test_size\": 1, \"train_size\": 3} plot_splits(X, y, groups, **cv_args) print_split_info(X, y, groups, **cv_args) Train indices: [0 1 2 3 4 5 6 7] Test indices: [8 9] Train length: 8 Test length: 2 Train groups: [0 1 1 1 1 2 2 2] Test groups: [3 3] Train group size: 3 Test group size: 1 Train group months: ['2021-01' '2021-02' '2021-02' '2021-02' '2021-02' '2021-03' '2021-03' '2021-03'] Test group months: ['2021-04' '2021-04'] Train indices: [1 2 3 4 5 6 7 8 9] Test indices: [10 11] Train length: 9 Test length: 2 Train groups: [1 1 1 1 2 2 2 3 3] Test groups: [4 4] Train group size: 3 Test group size: 1 Train group months: ['2021-02' '2021-02' '2021-02' '2021-02' '2021-03' '2021-03' '2021-03' '2021-04' '2021-04'] Test group months: ['2021-05' '2021-05'] Train indices: [ 5 6 7 8 9 10 11] Test indices: [12 13 14 15] Train length: 7 Test length: 4 Train groups: [2 2 2 3 3 4 4] Test groups: [5 5 5 5] Train group size: 3 Test group size: 1 Train group months: ['2021-03' '2021-03' '2021-03' '2021-04' '2021-04' '2021-05' '2021-05'] Test group months: ['2021-06' '2021-06' '2021-06' '2021-06'] Please note that if we specify the number of groups for both the training and the test set, the split size is determined automatically, and the number of splits naturally changes with the groups sizes. For example, increasing the number of training groups will naturally result in a lower number of splits as shown below. cv_args = {\"test_size\": 1, \"train_size\": 4} plot_splits(X, y, groups, **cv_args) Usage in CV The example below illustrates how we can use the time series splitter with scikit-learn, i.e., using cross_val_score : cv = GroupTimeSeriesSplit(**cv_args) clf = DummyClassifier(strategy=\"most_frequent\") scores = cross_val_score(clf, X, y, groups=groups, scoring=\"accuracy\", cv=cv) print_cv_info(cv, X, y, groups, clf, scores) Split number: 1 Train true target: [1 0 1 0 1 0 0 1] Train predicted target: [0 0 0 0 0 0 0 0] Test true target: [1 1] Test predicted target: [0 0] Accuracy: 0.0 Split number: 2 Train true target: [0 1 0 1 0 0 1 1 1] Train predicted target: [1 1 1 1 1 1 1 1 1] Test true target: [0 1] Test predicted target: [1 1] Accuracy: 0.5 Split number: 3 Train true target: [0 0 1 1 1 0 1] Train predicted target: [1 1 1 1 1 1 1] Test true target: [1 0 0 0] Test predicted target: [1 1 1 1] Accuracy: 0.25 Example 2 -- Multiple training groups (with number of splits specified) Now let's take a look at an example where we don't specify the number of training groups. Here, we will split the dataset with test size (2 groups) and a specified number of splits (3 groups), which is sufficient for calculating the training size automatically. cv_args = {\"test_size\": 2, \"n_splits\": 3} plot_splits(X, y, groups, **cv_args) print_split_info(X, y, groups, **cv_args) Train indices: [0 1 2 3 4] Test indices: [5 6 7 8 9] Train length: 5 Test length: 5 Train groups: [0 1 1 1 1] Test groups: [2 2 2 3 3] Train group size: 2 Test group size: 2 Train group months: ['2021-01' '2021-02' '2021-02' '2021-02' '2021-02'] Test group months: ['2021-03' '2021-03' '2021-03' '2021-04' '2021-04'] Train indices: [1 2 3 4 5 6 7] Test indices: [ 8 9 10 11] Train length: 7 Test length: 4 Train groups: [1 1 1 1 2 2 2] Test groups: [3 3 4 4] Train group size: 2 Test group size: 2 Train group months: ['2021-02' '2021-02' '2021-02' '2021-02' '2021-03' '2021-03' '2021-03'] Test group months: ['2021-04' '2021-04' '2021-05' '2021-05'] Train indices: [5 6 7 8 9] Test indices: [10 11 12 13 14 15] Train length: 5 Test length: 6 Train groups: [2 2 2 3 3] Test groups: [4 4 5 5 5 5] Train group size: 2 Test group size: 2 Train group months: ['2021-03' '2021-03' '2021-03' '2021-04' '2021-04'] Test group months: ['2021-05' '2021-05' '2021-06' '2021-06' '2021-06' '2021-06'] Usage in CV Again, let's take a look at how this looks in a scikit-learn context using cross_val_score : cv = GroupTimeSeriesSplit(**cv_args) clf = DummyClassifier(strategy=\"most_frequent\") scores = cross_val_score(clf, X, y, groups=groups, scoring=\"accuracy\", cv=cv) print_cv_info(cv, X, y, groups, clf, scores) Split number: 1 Train true target: [1 0 1 0 1] Train predicted target: [1 1 1 1 1] Test true target: [0 0 1 1 1] Test predicted target: [1 1 1 1 1] Accuracy: 0.6 Split number: 2 Train true target: [0 1 0 1 0 0 1] Train predicted target: [0 0 0 0 0 0 0] Test true target: [1 1 0 1] Test predicted target: [0 0 0 0] Accuracy: 0.25 Split number: 3 Train true target: [0 0 1 1 1] Train predicted target: [1 1 1 1 1] Test true target: [0 1 1 0 0 0] Test predicted target: [1 1 1 1 1 1] Accuracy: 0.33 Example 3 -- Defining the gap size between training and test datasets GroupTimeSeriesSplit let's you specify a gap size greater than 1 in order to skip a specified number of groups between training and test folds (the default gap size is 0). In the example below, we use a gap of 1 group to illustrate this. cv_args = {\"test_size\": 1, \"n_splits\": 3, \"gap_size\": 1} plot_splits(X, y, groups, **cv_args) print_split_info(X, y, groups, **cv_args) Train indices: [0 1 2 3 4] Test indices: [8 9] Train length: 5 Test length: 2 Train groups: [0 1 1 1 1] Test groups: [3 3] Train group size: 2 Test group size: 1 Train group months: ['2021-01' '2021-02' '2021-02' '2021-02' '2021-02'] Test group months: ['2021-04' '2021-04'] Train indices: [1 2 3 4 5 6 7] Test indices: [10 11] Train length: 7 Test length: 2 Train groups: [1 1 1 1 2 2 2] Test groups: [4 4] Train group size: 2 Test group size: 1 Train group months: ['2021-02' '2021-02' '2021-02' '2021-02' '2021-03' '2021-03' '2021-03'] Test group months: ['2021-05' '2021-05'] Train indices: [5 6 7 8 9] Test indices: [12 13 14 15] Train length: 5 Test length: 4 Train groups: [2 2 2 3 3] Test groups: [5 5 5 5] Train group size: 2 Test group size: 1 Train group months: ['2021-03' '2021-03' '2021-03' '2021-04' '2021-04'] Test group months: ['2021-06' '2021-06' '2021-06' '2021-06'] Usage in CV The example below shows how this looks like in a scikit-learn context using cross_val_score : cv = GroupTimeSeriesSplit(**cv_args) clf = DummyClassifier(strategy=\"most_frequent\") scores = cross_val_score(clf, X, y, groups=groups, scoring=\"accuracy\", cv=cv) print_cv_info(cv, X, y, groups, clf, scores) Split number: 1 Train true target: [1 0 1 0 1] Train predicted target: [1 1 1 1 1] Test true target: [1 1] Test predicted target: [1 1] Accuracy: 1.0 Split number: 2 Train true target: [0 1 0 1 0 0 1] Train predicted target: [0 0 0 0 0 0 0] Test true target: [0 1] Test predicted target: [0 0] Accuracy: 0.5 Split number: 3 Train true target: [0 0 1 1 1] Train predicted target: [1 1 1 1 1] Test true target: [1 0 0 0] Test predicted target: [1 1 1 1] Accuracy: 0.25 API GroupTimeSeriesSplit(test_size, train_size=None, n_splits=None, gap_size=0, shift_size=1, window_type='rolling') Group time series cross-validator. Parameters test_size : int Size of test dataset. train_size : int (default=None) Size of train dataset. n_splits : int (default=None) Number of the splits. gap_size : int (default=0) Gap size between train and test datasets. shift_size : int (default=1) Step to shift for the next fold. window_type : str (default=\"rolling\") Type of the window. Possible values: \"rolling\", \"expanding\". Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/GroupTimeSeriesSplit/ Methods get_n_splits(X=None, y=None, groups=None) Returns the number of splitting iterations in the cross-validator. Parameters X : object Always ignored, exists for compatibility. y : object Always ignored, exists for compatibility. groups : object Always ignored, exists for compatibility. Returns n_splits : int Returns the number of splitting iterations in the cross-validator. split(X, y=None, groups=None) Generate indices to split data into training and test set. Parameters X : array-like Training data. y : array-like (default=None) Always ignored, exists for compatibility. groups : array-like (default=None) Array with group names or sequence numbers. Yields train : ndarray The training set indices for that split. test : ndarray The testing set indices for that split.","title":"GroupTimeSeriesSplit: A scikit-learn compatible version of the time series validation with groups"},{"location":"user_guide/evaluate/GroupTimeSeriesSplit/#grouptimeseriessplit-a-scikit-learn-compatible-version-of-the-time-series-validation-with-groups","text":"A scikit-learn-compatible time series cross-validator that supports non-overlapping groups. from mlxtend.evaluate import GroupTimeSeriesSplit","title":"GroupTimeSeriesSplit: A scikit-learn compatible version of the time series validation with groups"},{"location":"user_guide/evaluate/GroupTimeSeriesSplit/#overview","text":"Time series tasks in machine learning require special type of validation, because the time order of the objects is important for a fairer evaluation of an ML model\u2019s quality. Also there can be different time units for splitting the data for different tasks - hours, days, months etc. Here, we use time series validation with support of the groups which can be flexibly configured along with other parameters: Test size Train size Number of splits Gap size Shift size Window type This GroupTimeSeriesSplit implementation is inspired by scikit-learn's TimeSeriesSplit but it has several advantages: GroupTimeSeriesSplit lets you group data as you want before splitting, while TimeSeriesSplit only considers the record level. It can be used for both holdout validation (n_splits=1) and cross-validation (n_splits>=2), whereas TimeSeriesSplit can be used only for the latter case. TimeSeriesSplit uses only an expanding window, while for this implementation you can choose between both rolling and expanding window types. GroupTimeSeriesSplit offers additional control for splitting using an additional shift size parameter. There are several features that need to be taken into account: GroupTimeSeriesSplit is compatible with sklearn-learn API. Numbers or custom non-numeric values can be used as groups However, groups should be consecutive Specifying the test size with either a) the train size or b) the number of splits is required parameters for splitting If full data can\u2019t be used with specific parameters, the most recent data is considered for splitting If splitting is impossible (e.g., there is not enough data to split) using specified parameters, an exception will be raised Before we illustrate the usage of GroupTimeSeriesSplit in the following examples below, let's set up a DummyClassifier that we will reuse in the following sections. Also, let's import the libraries we will be using in the following examples: import numpy as np import pandas as pd from sklearn.dummy import DummyClassifier from sklearn.model_selection import cross_val_score from mlxtend.evaluate.time_series import ( GroupTimeSeriesSplit, plot_splits, print_cv_info, print_split_info, )","title":"Overview"},{"location":"user_guide/evaluate/GroupTimeSeriesSplit/#prepare-sample-data","text":"For the following examples, we are creating sample dataset consisting of 16 training data points with corresponding targets.","title":"Prepare sample data"},{"location":"user_guide/evaluate/GroupTimeSeriesSplit/#features-and-targets","text":"Let's assume that we have one numeric feature and target for the binary classification task. data = [[0], [7], [6], [4], [4], [8], [0], [6], [2], [0], [5], [9], [7], [7], [7], [7]] target = [1, 0, 1, 0, 1, 0, 0, 1, 1, 1, 0, 1, 1, 0, 0, 0] X = pd.DataFrame(data, columns=[\"num_feature\"]) y = pd.Series(target, name=\"target\")","title":"Features and targets"},{"location":"user_guide/evaluate/GroupTimeSeriesSplit/#group-numbers","text":"We create 6 different groups so that the first training example belongs to group 0, the next 4 to group 1, and so forth. These groups do not have to be in ascending order (as in this dataset), but they must be consecutive. groups = np.array([0, 1, 1, 1, 1, 2, 2, 2, 3, 3, 4, 4, 5, 5, 5, 5]) groups array([0, 1, 1, 1, 1, 2, 2, 2, 3, 3, 4, 4, 5, 5, 5, 5]) Note that the following i another example of a correct group ordering (not sorted but consecutive): np.array([5, 5, 5, 5, 1, 1, 1, 1, 3, 3, 2, 2, 2, 4, 4, 0]) However, the example below shows an incorrect group ordering (not consecutive), which is not compatible with GroupTimeSeriesSplit : np.array([0, 1, 1, 1, 1, 2, 2, 2, 3, 3, 4, 4, 2, 2, 2, 2])","title":"Group numbers"},{"location":"user_guide/evaluate/GroupTimeSeriesSplit/#group-names-months","text":"We will add months as the index according to the specified groups for a more illustrative example. months_map = {i: f\"2021-0{i+1}\" for i in range(6)} months = np.array([months_map[group] for group in groups]) months array(['2021-01', '2021-02', '2021-02', '2021-02', '2021-02', '2021-03', '2021-03', '2021-03', '2021-04', '2021-04', '2021-05', '2021-05', '2021-06', '2021-06', '2021-06', '2021-06'], dtype='<U7') X = X.set_index(months)","title":"Group names (months)"},{"location":"user_guide/evaluate/GroupTimeSeriesSplit/#example-1-multiple-training-groups-with-train-size-specified","text":"Let's create a time series splitting with a training dataset that consists of 3 groups. And we will use 1 group for testing. In this case, the number of splits will be calculated automatically since both the training set and the test set sizes are specified. The first 3 rows in the visualization describe how each split is distributed among groups. The groups are visualized in the last row where each color represents different group. cv_args = {\"test_size\": 1, \"train_size\": 3} plot_splits(X, y, groups, **cv_args) print_split_info(X, y, groups, **cv_args) Train indices: [0 1 2 3 4 5 6 7] Test indices: [8 9] Train length: 8 Test length: 2 Train groups: [0 1 1 1 1 2 2 2] Test groups: [3 3] Train group size: 3 Test group size: 1 Train group months: ['2021-01' '2021-02' '2021-02' '2021-02' '2021-02' '2021-03' '2021-03' '2021-03'] Test group months: ['2021-04' '2021-04'] Train indices: [1 2 3 4 5 6 7 8 9] Test indices: [10 11] Train length: 9 Test length: 2 Train groups: [1 1 1 1 2 2 2 3 3] Test groups: [4 4] Train group size: 3 Test group size: 1 Train group months: ['2021-02' '2021-02' '2021-02' '2021-02' '2021-03' '2021-03' '2021-03' '2021-04' '2021-04'] Test group months: ['2021-05' '2021-05'] Train indices: [ 5 6 7 8 9 10 11] Test indices: [12 13 14 15] Train length: 7 Test length: 4 Train groups: [2 2 2 3 3 4 4] Test groups: [5 5 5 5] Train group size: 3 Test group size: 1 Train group months: ['2021-03' '2021-03' '2021-03' '2021-04' '2021-04' '2021-05' '2021-05'] Test group months: ['2021-06' '2021-06' '2021-06' '2021-06'] Please note that if we specify the number of groups for both the training and the test set, the split size is determined automatically, and the number of splits naturally changes with the groups sizes. For example, increasing the number of training groups will naturally result in a lower number of splits as shown below. cv_args = {\"test_size\": 1, \"train_size\": 4} plot_splits(X, y, groups, **cv_args)","title":"Example 1 -- Multiple training groups (with train size specified)"},{"location":"user_guide/evaluate/GroupTimeSeriesSplit/#usage-in-cv","text":"The example below illustrates how we can use the time series splitter with scikit-learn, i.e., using cross_val_score : cv = GroupTimeSeriesSplit(**cv_args) clf = DummyClassifier(strategy=\"most_frequent\") scores = cross_val_score(clf, X, y, groups=groups, scoring=\"accuracy\", cv=cv) print_cv_info(cv, X, y, groups, clf, scores) Split number: 1 Train true target: [1 0 1 0 1 0 0 1] Train predicted target: [0 0 0 0 0 0 0 0] Test true target: [1 1] Test predicted target: [0 0] Accuracy: 0.0 Split number: 2 Train true target: [0 1 0 1 0 0 1 1 1] Train predicted target: [1 1 1 1 1 1 1 1 1] Test true target: [0 1] Test predicted target: [1 1] Accuracy: 0.5 Split number: 3 Train true target: [0 0 1 1 1 0 1] Train predicted target: [1 1 1 1 1 1 1] Test true target: [1 0 0 0] Test predicted target: [1 1 1 1] Accuracy: 0.25","title":"Usage in CV"},{"location":"user_guide/evaluate/GroupTimeSeriesSplit/#example-2-multiple-training-groups-with-number-of-splits-specified","text":"Now let's take a look at an example where we don't specify the number of training groups. Here, we will split the dataset with test size (2 groups) and a specified number of splits (3 groups), which is sufficient for calculating the training size automatically. cv_args = {\"test_size\": 2, \"n_splits\": 3} plot_splits(X, y, groups, **cv_args) print_split_info(X, y, groups, **cv_args) Train indices: [0 1 2 3 4] Test indices: [5 6 7 8 9] Train length: 5 Test length: 5 Train groups: [0 1 1 1 1] Test groups: [2 2 2 3 3] Train group size: 2 Test group size: 2 Train group months: ['2021-01' '2021-02' '2021-02' '2021-02' '2021-02'] Test group months: ['2021-03' '2021-03' '2021-03' '2021-04' '2021-04'] Train indices: [1 2 3 4 5 6 7] Test indices: [ 8 9 10 11] Train length: 7 Test length: 4 Train groups: [1 1 1 1 2 2 2] Test groups: [3 3 4 4] Train group size: 2 Test group size: 2 Train group months: ['2021-02' '2021-02' '2021-02' '2021-02' '2021-03' '2021-03' '2021-03'] Test group months: ['2021-04' '2021-04' '2021-05' '2021-05'] Train indices: [5 6 7 8 9] Test indices: [10 11 12 13 14 15] Train length: 5 Test length: 6 Train groups: [2 2 2 3 3] Test groups: [4 4 5 5 5 5] Train group size: 2 Test group size: 2 Train group months: ['2021-03' '2021-03' '2021-03' '2021-04' '2021-04'] Test group months: ['2021-05' '2021-05' '2021-06' '2021-06' '2021-06' '2021-06']","title":"Example 2 -- Multiple training groups (with number of splits specified)"},{"location":"user_guide/evaluate/GroupTimeSeriesSplit/#usage-in-cv_1","text":"Again, let's take a look at how this looks in a scikit-learn context using cross_val_score : cv = GroupTimeSeriesSplit(**cv_args) clf = DummyClassifier(strategy=\"most_frequent\") scores = cross_val_score(clf, X, y, groups=groups, scoring=\"accuracy\", cv=cv) print_cv_info(cv, X, y, groups, clf, scores) Split number: 1 Train true target: [1 0 1 0 1] Train predicted target: [1 1 1 1 1] Test true target: [0 0 1 1 1] Test predicted target: [1 1 1 1 1] Accuracy: 0.6 Split number: 2 Train true target: [0 1 0 1 0 0 1] Train predicted target: [0 0 0 0 0 0 0] Test true target: [1 1 0 1] Test predicted target: [0 0 0 0] Accuracy: 0.25 Split number: 3 Train true target: [0 0 1 1 1] Train predicted target: [1 1 1 1 1] Test true target: [0 1 1 0 0 0] Test predicted target: [1 1 1 1 1 1] Accuracy: 0.33","title":"Usage in CV"},{"location":"user_guide/evaluate/GroupTimeSeriesSplit/#example-3-defining-the-gap-size-between-training-and-test-datasets","text":"GroupTimeSeriesSplit let's you specify a gap size greater than 1 in order to skip a specified number of groups between training and test folds (the default gap size is 0). In the example below, we use a gap of 1 group to illustrate this. cv_args = {\"test_size\": 1, \"n_splits\": 3, \"gap_size\": 1} plot_splits(X, y, groups, **cv_args) print_split_info(X, y, groups, **cv_args) Train indices: [0 1 2 3 4] Test indices: [8 9] Train length: 5 Test length: 2 Train groups: [0 1 1 1 1] Test groups: [3 3] Train group size: 2 Test group size: 1 Train group months: ['2021-01' '2021-02' '2021-02' '2021-02' '2021-02'] Test group months: ['2021-04' '2021-04'] Train indices: [1 2 3 4 5 6 7] Test indices: [10 11] Train length: 7 Test length: 2 Train groups: [1 1 1 1 2 2 2] Test groups: [4 4] Train group size: 2 Test group size: 1 Train group months: ['2021-02' '2021-02' '2021-02' '2021-02' '2021-03' '2021-03' '2021-03'] Test group months: ['2021-05' '2021-05'] Train indices: [5 6 7 8 9] Test indices: [12 13 14 15] Train length: 5 Test length: 4 Train groups: [2 2 2 3 3] Test groups: [5 5 5 5] Train group size: 2 Test group size: 1 Train group months: ['2021-03' '2021-03' '2021-03' '2021-04' '2021-04'] Test group months: ['2021-06' '2021-06' '2021-06' '2021-06']","title":"Example 3 -- Defining the gap size between training and test datasets"},{"location":"user_guide/evaluate/GroupTimeSeriesSplit/#usage-in-cv_2","text":"The example below shows how this looks like in a scikit-learn context using cross_val_score : cv = GroupTimeSeriesSplit(**cv_args) clf = DummyClassifier(strategy=\"most_frequent\") scores = cross_val_score(clf, X, y, groups=groups, scoring=\"accuracy\", cv=cv) print_cv_info(cv, X, y, groups, clf, scores) Split number: 1 Train true target: [1 0 1 0 1] Train predicted target: [1 1 1 1 1] Test true target: [1 1] Test predicted target: [1 1] Accuracy: 1.0 Split number: 2 Train true target: [0 1 0 1 0 0 1] Train predicted target: [0 0 0 0 0 0 0] Test true target: [0 1] Test predicted target: [0 0] Accuracy: 0.5 Split number: 3 Train true target: [0 0 1 1 1] Train predicted target: [1 1 1 1 1] Test true target: [1 0 0 0] Test predicted target: [1 1 1 1] Accuracy: 0.25","title":"Usage in CV"},{"location":"user_guide/evaluate/GroupTimeSeriesSplit/#api","text":"GroupTimeSeriesSplit(test_size, train_size=None, n_splits=None, gap_size=0, shift_size=1, window_type='rolling') Group time series cross-validator. Parameters test_size : int Size of test dataset. train_size : int (default=None) Size of train dataset. n_splits : int (default=None) Number of the splits. gap_size : int (default=0) Gap size between train and test datasets. shift_size : int (default=1) Step to shift for the next fold. window_type : str (default=\"rolling\") Type of the window. Possible values: \"rolling\", \"expanding\". Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/GroupTimeSeriesSplit/","title":"API"},{"location":"user_guide/evaluate/GroupTimeSeriesSplit/#methods","text":"get_n_splits(X=None, y=None, groups=None) Returns the number of splitting iterations in the cross-validator. Parameters X : object Always ignored, exists for compatibility. y : object Always ignored, exists for compatibility. groups : object Always ignored, exists for compatibility. Returns n_splits : int Returns the number of splitting iterations in the cross-validator. split(X, y=None, groups=None) Generate indices to split data into training and test set. Parameters X : array-like Training data. y : array-like (default=None) Always ignored, exists for compatibility. groups : array-like (default=None) Array with group names or sequence numbers. Yields train : ndarray The training set indices for that split. test : ndarray The testing set indices for that split.","title":"Methods"},{"location":"user_guide/evaluate/PredefinedHoldoutSplit/","text":"PredefinedHoldoutSplit: Utility for the holdout method compatible with scikit-learn Split a dataset into a train and validation subset for validation based on user-specified indices. from mlxtend.evaluate import PredefinedHoldoutSplit Overview The PredefinedHoldoutSplit class serves as an alternative to scikit-learn's KFold class, where the PredefinedHoldoutSplit class splits a dataset into training and a validation subsets without rotation, based on validation indices specified by the user. The PredefinedHoldoutSplit can be used as argument for cv parameters in scikit-learn's GridSearchCV etc. For performing a random split, see the related RandomHoldoutSplit class. Example 1 -- Iterating Over a PredefinedHoldoutSplit from mlxtend.evaluate import PredefinedHoldoutSplit from mlxtend.data import iris_data X, y = iris_data() h_iter = PredefinedHoldoutSplit(valid_indices=[0, 1, 99]) cnt = 0 for train_ind, valid_ind in h_iter.split(X, y): cnt += 1 print(cnt) 1 print(train_ind[:5]) print(valid_ind[:5]) [2 3 4 5 6] [ 0 1 99] Example 2 -- PredefinedHoldoutSplit in GridSearch from sklearn.model_selection import GridSearchCV from sklearn.neighbors import KNeighborsClassifier from mlxtend.evaluate import PredefinedHoldoutSplit from mlxtend.data import iris_data X, y = iris_data() params = {'n_neighbors': [1, 2, 3, 4, 5]} grid = GridSearchCV(KNeighborsClassifier(), param_grid=params, cv=PredefinedHoldoutSplit(valid_indices=[0, 1, 99])) grid.fit(X, y) GridSearchCV(cv=<mlxtend.evaluate.holdout.PredefinedHoldoutSplit object at 0x7fb300565610>, estimator=KNeighborsClassifier(), param_grid={'n_neighbors': [1, 2, 3, 4, 5]}) API PredefinedHoldoutSplit(valid_indices) Train/Validation set splitter for sklearn's GridSearchCV etc. Uses user-specified train/validation set indices to split a dataset into train/validation sets using user-defined or random indices. Parameters valid_indices : array-like, shape (num_examples,) Indices of the training examples in the training set to be used for validation. All other indices in the training set are used to for a training subset for model fitting. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/PredefinedHoldoutSplit/ Methods get_n_splits(X=None, y=None, groups=None) Returns the number of splitting iterations in the cross-validator Parameters X : object Always ignored, exists for compatibility. y : object Always ignored, exists for compatibility. groups : object Always ignored, exists for compatibility. Returns n_splits : 1 Returns the number of splitting iterations in the cross-validator. Always returns 1. split(X, y, groups=None) Generate indices to split data into training and test set. Parameters X : array-like, shape (num_examples, num_features) Training data, where num_examples is the number of examples and num_features is the number of features. y : array-like, shape (num_examples,) The target variable for supervised learning problems. Stratification is done based on the y labels. groups : object Always ignored, exists for compatibility. Yields train_index : ndarray The training set indices for that split. valid_index : ndarray The validation set indices for that split.","title":"PredefinedHoldoutSplit: Utility for the holdout method compatible with scikit-learn"},{"location":"user_guide/evaluate/PredefinedHoldoutSplit/#predefinedholdoutsplit-utility-for-the-holdout-method-compatible-with-scikit-learn","text":"Split a dataset into a train and validation subset for validation based on user-specified indices. from mlxtend.evaluate import PredefinedHoldoutSplit","title":"PredefinedHoldoutSplit: Utility for the holdout method compatible with scikit-learn"},{"location":"user_guide/evaluate/PredefinedHoldoutSplit/#overview","text":"The PredefinedHoldoutSplit class serves as an alternative to scikit-learn's KFold class, where the PredefinedHoldoutSplit class splits a dataset into training and a validation subsets without rotation, based on validation indices specified by the user. The PredefinedHoldoutSplit can be used as argument for cv parameters in scikit-learn's GridSearchCV etc. For performing a random split, see the related RandomHoldoutSplit class.","title":"Overview"},{"location":"user_guide/evaluate/PredefinedHoldoutSplit/#example-1-iterating-over-a-predefinedholdoutsplit","text":"from mlxtend.evaluate import PredefinedHoldoutSplit from mlxtend.data import iris_data X, y = iris_data() h_iter = PredefinedHoldoutSplit(valid_indices=[0, 1, 99]) cnt = 0 for train_ind, valid_ind in h_iter.split(X, y): cnt += 1 print(cnt) 1 print(train_ind[:5]) print(valid_ind[:5]) [2 3 4 5 6] [ 0 1 99]","title":"Example 1 -- Iterating Over a PredefinedHoldoutSplit"},{"location":"user_guide/evaluate/PredefinedHoldoutSplit/#example-2-predefinedholdoutsplit-in-gridsearch","text":"from sklearn.model_selection import GridSearchCV from sklearn.neighbors import KNeighborsClassifier from mlxtend.evaluate import PredefinedHoldoutSplit from mlxtend.data import iris_data X, y = iris_data() params = {'n_neighbors': [1, 2, 3, 4, 5]} grid = GridSearchCV(KNeighborsClassifier(), param_grid=params, cv=PredefinedHoldoutSplit(valid_indices=[0, 1, 99])) grid.fit(X, y) GridSearchCV(cv=<mlxtend.evaluate.holdout.PredefinedHoldoutSplit object at 0x7fb300565610>, estimator=KNeighborsClassifier(), param_grid={'n_neighbors': [1, 2, 3, 4, 5]})","title":"Example 2 -- PredefinedHoldoutSplit in GridSearch"},{"location":"user_guide/evaluate/PredefinedHoldoutSplit/#api","text":"PredefinedHoldoutSplit(valid_indices) Train/Validation set splitter for sklearn's GridSearchCV etc. Uses user-specified train/validation set indices to split a dataset into train/validation sets using user-defined or random indices. Parameters valid_indices : array-like, shape (num_examples,) Indices of the training examples in the training set to be used for validation. All other indices in the training set are used to for a training subset for model fitting. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/PredefinedHoldoutSplit/","title":"API"},{"location":"user_guide/evaluate/PredefinedHoldoutSplit/#methods","text":"get_n_splits(X=None, y=None, groups=None) Returns the number of splitting iterations in the cross-validator Parameters X : object Always ignored, exists for compatibility. y : object Always ignored, exists for compatibility. groups : object Always ignored, exists for compatibility. Returns n_splits : 1 Returns the number of splitting iterations in the cross-validator. Always returns 1. split(X, y, groups=None) Generate indices to split data into training and test set. Parameters X : array-like, shape (num_examples, num_features) Training data, where num_examples is the number of examples and num_features is the number of features. y : array-like, shape (num_examples,) The target variable for supervised learning problems. Stratification is done based on the y labels. groups : object Always ignored, exists for compatibility. Yields train_index : ndarray The training set indices for that split. valid_index : ndarray The validation set indices for that split.","title":"Methods"},{"location":"user_guide/evaluate/RandomHoldoutSplit/","text":"RandomHoldoutSplit: split a dataset into a train and validation subset for validation Randomly split a dataset into a train and validation subset for validation. from mlxtend.evaluate import RandomHoldoutSplit Overview The RandomHoldoutSplit class serves as an alternative to scikit-learn's KFold class, where the RandomHoldoutSplit class splits a dataset into training and a validation subsets without rotation. The RandomHoldoutSplit can be used as argument for cv parameters in scikit-learn's GridSearchCV etc. The term \"random\" in RandomHoldoutSplit comes from the fact that the split is specified by the random_seed rather than specifying the training and validation set indices manually as in the PredefinedHoldoutSplit class in mlxtend. Example 1 -- Iterating Over a RandomHoldoutSplit from mlxtend.evaluate import RandomHoldoutSplit from mlxtend.data import iris_data X, y = iris_data() h_iter = RandomHoldoutSplit(valid_size=0.3, random_seed=123) cnt = 0 for train_ind, valid_ind in h_iter.split(X, y): cnt += 1 print(cnt) 1 print(train_ind[:5]) print(valid_ind[:5]) [ 60 16 88 130 6] [ 72 125 80 86 117] Example 2 -- RandomHoldoutSplit in GridSearch from sklearn.model_selection import GridSearchCV from sklearn.neighbors import KNeighborsClassifier from mlxtend.evaluate import RandomHoldoutSplit from mlxtend.data import iris_data X, y = iris_data() params = {'n_neighbors': [1, 2, 3, 4, 5]} grid = GridSearchCV(KNeighborsClassifier(), param_grid=params, cv=RandomHoldoutSplit(valid_size=0.3, random_seed=123)) grid.fit(X, y) GridSearchCV(cv=<mlxtend.evaluate.holdout.RandomHoldoutSplit object at 0x7fae707f6610>, estimator=KNeighborsClassifier(), param_grid={'n_neighbors': [1, 2, 3, 4, 5]}) API RandomHoldoutSplit(valid_size=0.5, random_seed=None, stratify=False) Train/Validation set splitter for sklearn's GridSearchCV etc. Provides train/validation set indices to split a dataset into train/validation sets using random indices. Parameters valid_size : float (default: 0.5) Proportion of examples that being assigned as validation examples. 1- valid_size will then automatically be assigned as training set examples. random_seed : int (default: None) The random seed for splitting the data into training and validation set partitions. stratify : bool (default: False) True or False, whether to perform a stratified split or not Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/RandomHoldoutSplit/ Methods get_n_splits(X=None, y=None, groups=None) Returns the number of splitting iterations in the cross-validator Parameters X : object Always ignored, exists for compatibility. y : object Always ignored, exists for compatibility. groups : object Always ignored, exists for compatibility. Returns n_splits : 1 Returns the number of splitting iterations in the cross-validator. Always returns 1. split(X, y, groups=None) Generate indices to split data into training and test set. Parameters X : array-like, shape (num_examples, num_features) Training data, where num_examples is the number of training examples and num_features is the number of features. y : array-like, shape (num_examples,) The target variable for supervised learning problems. Stratification is done based on the y labels. groups : object Always ignored, exists for compatibility. Yields train_index : ndarray The training set indices for that split. valid_index : ndarray The validation set indices for that split.","title":"RandomHoldoutSplit: split a dataset into a train and validation subset for validation"},{"location":"user_guide/evaluate/RandomHoldoutSplit/#randomholdoutsplit-split-a-dataset-into-a-train-and-validation-subset-for-validation","text":"Randomly split a dataset into a train and validation subset for validation. from mlxtend.evaluate import RandomHoldoutSplit","title":"RandomHoldoutSplit: split a dataset into a train and validation subset for validation"},{"location":"user_guide/evaluate/RandomHoldoutSplit/#overview","text":"The RandomHoldoutSplit class serves as an alternative to scikit-learn's KFold class, where the RandomHoldoutSplit class splits a dataset into training and a validation subsets without rotation. The RandomHoldoutSplit can be used as argument for cv parameters in scikit-learn's GridSearchCV etc. The term \"random\" in RandomHoldoutSplit comes from the fact that the split is specified by the random_seed rather than specifying the training and validation set indices manually as in the PredefinedHoldoutSplit class in mlxtend.","title":"Overview"},{"location":"user_guide/evaluate/RandomHoldoutSplit/#example-1-iterating-over-a-randomholdoutsplit","text":"from mlxtend.evaluate import RandomHoldoutSplit from mlxtend.data import iris_data X, y = iris_data() h_iter = RandomHoldoutSplit(valid_size=0.3, random_seed=123) cnt = 0 for train_ind, valid_ind in h_iter.split(X, y): cnt += 1 print(cnt) 1 print(train_ind[:5]) print(valid_ind[:5]) [ 60 16 88 130 6] [ 72 125 80 86 117]","title":"Example 1 -- Iterating Over a RandomHoldoutSplit"},{"location":"user_guide/evaluate/RandomHoldoutSplit/#example-2-randomholdoutsplit-in-gridsearch","text":"from sklearn.model_selection import GridSearchCV from sklearn.neighbors import KNeighborsClassifier from mlxtend.evaluate import RandomHoldoutSplit from mlxtend.data import iris_data X, y = iris_data() params = {'n_neighbors': [1, 2, 3, 4, 5]} grid = GridSearchCV(KNeighborsClassifier(), param_grid=params, cv=RandomHoldoutSplit(valid_size=0.3, random_seed=123)) grid.fit(X, y) GridSearchCV(cv=<mlxtend.evaluate.holdout.RandomHoldoutSplit object at 0x7fae707f6610>, estimator=KNeighborsClassifier(), param_grid={'n_neighbors': [1, 2, 3, 4, 5]})","title":"Example 2 -- RandomHoldoutSplit in GridSearch"},{"location":"user_guide/evaluate/RandomHoldoutSplit/#api","text":"RandomHoldoutSplit(valid_size=0.5, random_seed=None, stratify=False) Train/Validation set splitter for sklearn's GridSearchCV etc. Provides train/validation set indices to split a dataset into train/validation sets using random indices. Parameters valid_size : float (default: 0.5) Proportion of examples that being assigned as validation examples. 1- valid_size will then automatically be assigned as training set examples. random_seed : int (default: None) The random seed for splitting the data into training and validation set partitions. stratify : bool (default: False) True or False, whether to perform a stratified split or not Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/RandomHoldoutSplit/","title":"API"},{"location":"user_guide/evaluate/RandomHoldoutSplit/#methods","text":"get_n_splits(X=None, y=None, groups=None) Returns the number of splitting iterations in the cross-validator Parameters X : object Always ignored, exists for compatibility. y : object Always ignored, exists for compatibility. groups : object Always ignored, exists for compatibility. Returns n_splits : 1 Returns the number of splitting iterations in the cross-validator. Always returns 1. split(X, y, groups=None) Generate indices to split data into training and test set. Parameters X : array-like, shape (num_examples, num_features) Training data, where num_examples is the number of training examples and num_features is the number of features. y : array-like, shape (num_examples,) The target variable for supervised learning problems. Stratification is done based on the y labels. groups : object Always ignored, exists for compatibility. Yields train_index : ndarray The training set indices for that split. valid_index : ndarray The validation set indices for that split.","title":"Methods"},{"location":"user_guide/evaluate/accuracy_score/","text":"accuracy_score: Computing standard, balanced, and per-class accuracy A function for computing for computing basic classifcation accuracy, per-class accuracy, and average per-class accuracy. from mlxtend.evaluate import accuracy_score Example 1 -- Standard Accuracy The \"overall\" accuracy is defined as the number of correct predictions ( true positives TP and true negatives TN) over all samples n : ACC = \\frac{TP + TN}{n} import numpy as np from mlxtend.evaluate import accuracy_score y_targ = [0, 0, 0, 1, 1, 1, 2, 2, 2] y_pred = [1, 0, 0, 0, 1, 2, 0, 2, 2] accuracy_score(y_targ, y_pred) 0.5555555555555556 Example 2 -- Per-Class Accuracy The per-class accuracy is the accuracy of one class (defined as the pos_label ) versus all remaining datapoints in the dataset. import numpy as np from mlxtend.evaluate import accuracy_score y_targ = [0, 0, 0, 1, 1, 1, 2, 2, 2] y_pred = [1, 0, 0, 0, 1, 2, 0, 2, 2] std_acc = accuracy_score(y_targ, y_pred) bin_acc = accuracy_score(y_targ, y_pred, method='binary', pos_label=1) print(f'Standard accuracy: {std_acc*100:.2f}%') print(f'Class 1 accuracy: {bin_acc*100:.2f}%') Standard accuracy: 55.56% Class 1 accuracy: 66.67% Example 3 -- Average Per-Class Accuracy Overview The \"overall\" accuracy is defined as the number of correct predictions ( true positives TP and true negatives TN) over all samples n : ACC = \\frac{TP + TN}{n} in a binary class setting: In a multi-class setting, we can generalize the computation of the accuracy as the fraction of all true predictions (the diagonal) over all samples n. ACC = \\frac{T}{n} Considering a multi-class problem with 3 classes (C0, C1, C2) let's assume our model made the following predictions: We compute the accuracy as: ACC = \\frac{3 + 50 + 18}{90} \\approx 0.79 Now, in order to compute the average per-class accuracy , we compute the binary accuracy for each class label separately; i.e., if class 1 is the positive class, class 0 and 2 are both considered the negative class. APC\\;ACC = \\frac{83/90 + 71/90 + 78/90}{3} \\approx 0.86 import numpy as np from mlxtend.evaluate import accuracy_score y_targ = [0, 0, 0, 1, 1, 1, 2, 0, 0] y_pred = [1, 0, 0, 0, 1, 2, 0, 2, 1] std_acc = accuracy_score(y_targ, y_pred) bin_acc = accuracy_score(y_targ, y_pred, method='binary', pos_label=1) avg_acc = accuracy_score(y_targ, y_pred, method='average') print(f'Standard accuracy: {std_acc*100:.2f}%') print(f'Class 1 accuracy: {bin_acc*100:.2f}%') print(f'Average per-class accuracy: {avg_acc*100:.2f}%') Standard accuracy: 33.33% Class 1 accuracy: 55.56% Average per-class accuracy: 55.56% References [1] S. Raschka. An overview of general performance metrics of binary classifier systems . Computing Research Repository (CoRR), abs/1410.5330, 2014. [2] Cyril Goutte and Eric Gaussier. A probabilistic interpretation of precision, recall and f-score, with implication for evaluation . In Advances in Information Retrieval, pages 345\u2013359. Springer, 2005. [3] Brian W Matthews. Comparison of the predicted and observed secondary structure of T4 phage lysozyme . Biochimica et Biophysica Acta (BBA)- Protein Structure, 405(2):442\u2013451, 1975. API accuracy_score(y_target, y_predicted, method='standard', pos_label=1, normalize=True) General accuracy function for supervised learning. Parameters y_target : array-like, shape=[n_values] True class labels or target values. y_predicted : array-like, shape=[n_values] Predicted class labels or target values. method : str, 'standard' by default. The chosen method for accuracy computation. If set to 'standard', computes overall accuracy. If set to 'binary', computes accuracy for class pos_label. If set to 'average', computes average per-class (balanced) accuracy. If set to 'balanced', computes the scikit-learn-style balanced accuracy. pos_label : str or int, 1 by default. The class whose accuracy score is to be reported. Used only when method is set to 'binary' normalize : bool, True by default. If True, returns fraction of correctly classified samples. If False, returns number of correctly classified samples. Returns score: float Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/accuracy_score/","title":"accuracy_score: Computing standard, balanced, and per-class accuracy"},{"location":"user_guide/evaluate/accuracy_score/#accuracy_score-computing-standard-balanced-and-per-class-accuracy","text":"A function for computing for computing basic classifcation accuracy, per-class accuracy, and average per-class accuracy. from mlxtend.evaluate import accuracy_score","title":"accuracy_score: Computing standard, balanced, and per-class accuracy"},{"location":"user_guide/evaluate/accuracy_score/#example-1-standard-accuracy","text":"The \"overall\" accuracy is defined as the number of correct predictions ( true positives TP and true negatives TN) over all samples n : ACC = \\frac{TP + TN}{n} import numpy as np from mlxtend.evaluate import accuracy_score y_targ = [0, 0, 0, 1, 1, 1, 2, 2, 2] y_pred = [1, 0, 0, 0, 1, 2, 0, 2, 2] accuracy_score(y_targ, y_pred) 0.5555555555555556","title":"Example 1 -- Standard Accuracy"},{"location":"user_guide/evaluate/accuracy_score/#example-2-per-class-accuracy","text":"The per-class accuracy is the accuracy of one class (defined as the pos_label ) versus all remaining datapoints in the dataset. import numpy as np from mlxtend.evaluate import accuracy_score y_targ = [0, 0, 0, 1, 1, 1, 2, 2, 2] y_pred = [1, 0, 0, 0, 1, 2, 0, 2, 2] std_acc = accuracy_score(y_targ, y_pred) bin_acc = accuracy_score(y_targ, y_pred, method='binary', pos_label=1) print(f'Standard accuracy: {std_acc*100:.2f}%') print(f'Class 1 accuracy: {bin_acc*100:.2f}%') Standard accuracy: 55.56% Class 1 accuracy: 66.67%","title":"Example 2 -- Per-Class Accuracy"},{"location":"user_guide/evaluate/accuracy_score/#example-3-average-per-class-accuracy","text":"","title":"Example 3 -- Average Per-Class Accuracy"},{"location":"user_guide/evaluate/accuracy_score/#overview","text":"The \"overall\" accuracy is defined as the number of correct predictions ( true positives TP and true negatives TN) over all samples n : ACC = \\frac{TP + TN}{n} in a binary class setting: In a multi-class setting, we can generalize the computation of the accuracy as the fraction of all true predictions (the diagonal) over all samples n. ACC = \\frac{T}{n} Considering a multi-class problem with 3 classes (C0, C1, C2) let's assume our model made the following predictions: We compute the accuracy as: ACC = \\frac{3 + 50 + 18}{90} \\approx 0.79 Now, in order to compute the average per-class accuracy , we compute the binary accuracy for each class label separately; i.e., if class 1 is the positive class, class 0 and 2 are both considered the negative class. APC\\;ACC = \\frac{83/90 + 71/90 + 78/90}{3} \\approx 0.86 import numpy as np from mlxtend.evaluate import accuracy_score y_targ = [0, 0, 0, 1, 1, 1, 2, 0, 0] y_pred = [1, 0, 0, 0, 1, 2, 0, 2, 1] std_acc = accuracy_score(y_targ, y_pred) bin_acc = accuracy_score(y_targ, y_pred, method='binary', pos_label=1) avg_acc = accuracy_score(y_targ, y_pred, method='average') print(f'Standard accuracy: {std_acc*100:.2f}%') print(f'Class 1 accuracy: {bin_acc*100:.2f}%') print(f'Average per-class accuracy: {avg_acc*100:.2f}%') Standard accuracy: 33.33% Class 1 accuracy: 55.56% Average per-class accuracy: 55.56%","title":"Overview"},{"location":"user_guide/evaluate/accuracy_score/#references","text":"[1] S. Raschka. An overview of general performance metrics of binary classifier systems . Computing Research Repository (CoRR), abs/1410.5330, 2014. [2] Cyril Goutte and Eric Gaussier. A probabilistic interpretation of precision, recall and f-score, with implication for evaluation . In Advances in Information Retrieval, pages 345\u2013359. Springer, 2005. [3] Brian W Matthews. Comparison of the predicted and observed secondary structure of T4 phage lysozyme . Biochimica et Biophysica Acta (BBA)- Protein Structure, 405(2):442\u2013451, 1975.","title":"References"},{"location":"user_guide/evaluate/accuracy_score/#api","text":"accuracy_score(y_target, y_predicted, method='standard', pos_label=1, normalize=True) General accuracy function for supervised learning. Parameters y_target : array-like, shape=[n_values] True class labels or target values. y_predicted : array-like, shape=[n_values] Predicted class labels or target values. method : str, 'standard' by default. The chosen method for accuracy computation. If set to 'standard', computes overall accuracy. If set to 'binary', computes accuracy for class pos_label. If set to 'average', computes average per-class (balanced) accuracy. If set to 'balanced', computes the scikit-learn-style balanced accuracy. pos_label : str or int, 1 by default. The class whose accuracy score is to be reported. Used only when method is set to 'binary' normalize : bool, True by default. If True, returns fraction of correctly classified samples. If False, returns number of correctly classified samples. Returns score: float Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/accuracy_score/","title":"API"},{"location":"user_guide/evaluate/bias_variance_decomp/","text":"bias_variance_decomp: Bias-variance decomposition for classification and regression losses Bias variance decomposition of machine learning algorithms for various loss functions. from mlxtend.evaluate import bias_variance_decomp Overview Often, researchers use the terms bias and variance or \"bias-variance tradeoff\" to describe the performance of a model -- i.e., you may stumble upon talks, books, or articles where people say that a model has a high variance or high bias. So, what does that mean? In general, we might say that \"high variance\" is proportional to overfitting, and \"high bias\" is proportional to underfitting. Anyways, why are we attempting to do this bias-variance decomposition in the first place? The decomposition of the loss into bias and variance helps us understand learning algorithms, as these concepts are correlated to underfitting and overfitting. To use the more formal terms for bias and variance, assume we have a point estimator \\hat{\\theta} of some parameter or function \\theta . Then, the bias is commonly defined as the difference between the expected value of the estimator and the parameter that we want to estimate: \\text{Bias} = E[\\hat{\\theta}] - \\theta. If the bias is larger than zero, we also say that the estimator is positively biased, if the bias is smaller than zero, the estimator is negatively biased, and if the bias is exactly zero, the estimator is unbiased. Similarly, we define the variance as the difference between the expected value of the squared estimator minus the squared expectation of the estimator: \\text{Var}(\\hat{\\theta}) = E\\big[\\hat{\\theta}^2\\big] - \\bigg(E\\big[\\hat{\\theta}\\big]\\bigg)^2. Note that in the context of this lecture, it will be more convenient to write the variance in its alternative form: \\text{Var}(\\hat{\\theta}) = E[(E[{\\hat{\\theta}}] - \\hat{\\theta})^2]. To illustrate the concept further in context of machine learning ... Suppose there is an unknown target function or \"true function\" to which we do want to approximate. Now, suppose we have different training sets drawn from an unknown distribution defined as \"true function + noise.\" The following plot shows different linear regression models, each fit to a different training set. None of these hypotheses approximate the true function well, except at two points (around x=-10 and x=6). Here, we can say that the bias is large because the difference between the true value and the predicted value, on average (here, average means \"expectation of the training sets\" not \"expectation over examples in the training set\"), is large: The next plot shows different unpruned decision tree models, each fit to a different training set. Note that these hypotheses fit the training data very closely. However, if we would consider the expectation over training sets, the average hypothesis would fit the true function perfectly (given that the noise is unbiased and has an expected value of 0). As we can see, the variance is very large, since on average, a prediction differs a lot from the expectation value of the prediction: Bias-Variance Decomposition of the Squared Loss We can decompose a loss function such as the squared loss into three terms, a variance, bias, and a noise term (and the same is true for the decomposition of the 0-1 loss later). However, for simplicity, we will ignore the noise term. Before we introduce the bias-variance decomposition of the 0-1 loss for classification, let us start with the decomposition of the squared loss as an easy warm-up exercise to get familiar with the overall concept. The previous section already listed the common formal definitions of bias and variance, however, let us define them again for convenience: \\text{Bias}(\\hat{\\theta}) = E[\\hat{\\theta}] - \\theta, \\quad \\text{Var}(\\hat{\\theta}) = E[(E[{\\hat{\\theta}}] - \\hat{\\theta})^2]. Recall that in the context of these machine learning lecture (notes), we defined the true or target function as y = f(x) , the predicted target value as \\hat{y} = \\hat{f}(x) = h(x) , and the squared loss as S = (y - \\hat{y})^2 . (I use S here because it will be easier to tell it apart from the E , which we use for the expectation in this lecture.) Note that unless noted otherwise, the expectation is over training sets! To get started with the squared error loss decomposition into bias and variance, let use do some algebraic manipulation, i.e., adding and subtracting the expected value of \\hat{y} and then expanding the expression using the quadratic formula (a+b)^2 = a^2 + b^2 + 2ab) : \\begin{equation} \\begin{split} S = (y - \\hat{y})^2 \\\\ (y - \\hat{y})^2 &= (y - E[{\\hat{y}}] + E[{\\hat{y}}] - \\hat{y})^2 \\\\ &= (y-E[{\\hat{y}}])^2 + (E[{\\hat{y}}] - y)^2 + 2(y - E[\\hat{y}])(E[\\hat{y}] - \\hat{y}). \\end{split} \\end{equation} Next, we just use the expectation on both sides, and we are already done: \\begin{align} E[S] &= E[(y - \\hat{y})^2] \\\\ E[(y - \\hat{y})^2] &= (y-E[{\\hat{y}}])^2 + E[(E[{\\hat{y}}] - \\hat{y})^2]\\\\ &= \\text{[Bias]}^2 + \\text{Variance}. \\end{align} You may wonder what happened to the \" 2ab \" term ( 2(y - E[\\hat{y}])(E[\\hat{y}] - \\hat{y}) ) when we used the expectation. It turns that it evaluates to zero and hence vanishes from the equation, which can be shown as follows: \\begin{align} E[2(y - E[{\\hat{y}}])(E[{\\hat{y}}] - \\hat{y})] &= 2 E[(y - E[{\\hat{y}}])(E[{\\hat{y}}] - \\hat{y})] \\\\ &= 2(y - E[{\\hat{y}}])E[(E[{\\hat{y}}] - \\hat{y})] \\\\ &= 2(y - E[{\\hat{y}}])(E[E[{\\hat{y}}]] - E[\\hat{y}])\\\\ &= 2(y - E[{\\hat{y}}])(E[{\\hat{y}}] - E[{\\hat{y}}]) \\\\ &= 0. \\end{align} So, this is the canonical decomposition of the squared error loss into bias and variance. The next section will discuss some approaches that have been made to decompose the 0-1 loss that we commonly use for classification accuracy or error. The following figure is a sketch of variance and bias in relation to the training error and generalization error -- how high variance related to overfitting, and how large bias relates to underfitting: Bias-Variance Decomposition of the 0-1 Loss Note that decomposing the 0-1 loss into bias and variance components is not as straight-forward as for the squared error loss. To quote Pedro Domingos, a well-known machine learning researcher and professor at University of Washington: \"several authors have proposed bias-variance decompositions related to zero-one loss (Kong & Dietterich, 1995; Breiman, 1996b; Kohavi & Wolpert, 1996; Tibshirani, 1996; Friedman, 1997). However, each of these decompositions has significant shortcomings.\". [1] In fact, the paper this quote was taken from may offer the most intuitive and general formulation at this point. However, we will first, for simplicity, go over Kong & Dietterich formulation [2] of the 0-1 loss decomposition, which is the same as Domingos's but excluding the noise term (for simplicity). The table below summarizes the relevant terms we used for the squared loss in relation to the 0-1 loss. Recall that the 0-1 loss, L , is 0 if a class label is predicted correctly, and one otherwise. The main prediction for the squared error loss is simply the average over the predictions E[\\hat{y}] (the expectation is over training sets), for the 0-1 loss Kong & Dietterich and Domingos defined it as the mode. I.e., if a model predicts the label one more than 50% of the time (considering all possible training sets), then the main prediction is 1, and 0 otherwise. - Squared Loss 0-1 Loss Single loss (y - \\hat{y})^2 L(y, \\hat{y}) Expected loss E[(y - \\hat{y})^2] E[L(y, \\hat{y})] Main prediction E[\\hat{y}] mean (average) mode Bias ^2 (y-E[{\\hat{y}}])^2 L(y, E[\\hat{y}]) Variance E[(E[{\\hat{y}}] - \\hat{y})^2] E[L(\\hat{y}, E[\\hat{y}])] Hence, as result from using the mode to define the main prediction of the 0-1 loss, the bias is 1 if the main prediction does not agree with the true label y , and 0 otherwise: Bias = \\begin{cases} 1 \\text{ if } y \\neq E[{\\hat{y}}], \\\\ 0 \\text{ otherwise}. \\end{cases} The variance of the 0-1 loss is defined as the probability that the predicted label does not match the main prediction: Variance = P(\\hat{y} \\neq E[\\hat{{y}}]). Next, let us take a look at what happens to the loss if the bias is 0. Given the general definition of the loss, loss = bias + variance, if the bias is 0, then we define the loss as the variance: Loss = 0 + Variance = Loss = P(\\hat{y} \\neq y) = Variance = P(\\hat{y} \\neq E[\\hat{{y}}]). In other words, if a model has zero bias, it's loss is entirely defined by the variance, which is intuitive if we think of variance in the context of being proportional overfitting. The more surprising scenario is if the bias is equal to 1. If the bias is equal to 1, as explained by Pedro Domingos, the increasing the variance can decrease the loss, which is an interesting observation. This can be seen by first rewriting the 0-1 loss function as Loss = P(\\hat{y} \\neq y) = 1 - P(\\hat{y} = y). (Note that we have not done anything new, yet.) Now, if we look at the previous equation of the bias, if the bias is 1, we have y \\neq E[{\\hat{y}}] . If y is not equal to the main prediction, but y is also is equal to \\hat{y} , then \\hat{y} must be equal to the main prediction. Using the \"inverse\" (\"1 minus\"), we can then write the loss as Loss = P(\\hat{y} \\neq y) = 1 - P(\\hat{y} = y) = 1 - P(\\hat{y} \\neq E[{\\hat{y}}]). Since the bias is 1, the loss is hence defined as \"loss = bias - variance\" if the bias is 1 (or \"loss = 1 - variance\"). This might be quite unintuitive at first, but the explanations Kong, Dietterich, and Domingos offer was that if a model has a very high bias such that it main prediction is always wrong, increasing the variance can be beneficial, since increasing the variance would push the decision boundary, which might lead to some correct predictions just by chance then. In other words, for scenarios with high bias, increasing the variance can improve (decrease) the loss! References [1] Domingos, Pedro. \"A unified bias-variance decomposition.\" Proceedings of 17th International Conference on Machine Learning. 2000. [2] Dietterich, Thomas G., and Eun Bae Kong. Machine learning bias, statistical bias, and statistical variance of decision tree algorithms. Technical report, Department of Computer Science, Oregon State University, 1995. Example 1 -- Bias Variance Decomposition of a Decision Tree Classifier from mlxtend.evaluate import bias_variance_decomp from sklearn.tree import DecisionTreeClassifier from mlxtend.data import iris_data from sklearn.model_selection import train_test_split X, y = iris_data() X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=123, shuffle=True, stratify=y) tree = DecisionTreeClassifier(random_state=123) avg_expected_loss, avg_bias, avg_var = bias_variance_decomp( tree, X_train, y_train, X_test, y_test, loss='0-1_loss', random_seed=123) print('Average expected loss: %.3f' % avg_expected_loss) print('Average bias: %.3f' % avg_bias) print('Average variance: %.3f' % avg_var) Average expected loss: 0.062 Average bias: 0.022 Average variance: 0.040 For comparison, the bias-variance decomposition of a bagging classifier, which should intuitively have a lower variance compared than a single decision tree: from sklearn.ensemble import BaggingClassifier tree = DecisionTreeClassifier(random_state=123) bag = BaggingClassifier(base_estimator=tree, n_estimators=100, random_state=123) avg_expected_loss, avg_bias, avg_var = bias_variance_decomp( bag, X_train, y_train, X_test, y_test, loss='0-1_loss', random_seed=123) print('Average expected loss: %.3f' % avg_expected_loss) print('Average bias: %.3f' % avg_bias) print('Average variance: %.3f' % avg_var) Average expected loss: 0.048 Average bias: 0.022 Average variance: 0.026 Example 2 -- Bias Variance Decomposition of a Decision Tree Regressor from mlxtend.evaluate import bias_variance_decomp from sklearn.tree import DecisionTreeRegressor from mlxtend.data import boston_housing_data from sklearn.model_selection import train_test_split X, y = boston_housing_data() X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=123, shuffle=True) tree = DecisionTreeRegressor(random_state=123) avg_expected_loss, avg_bias, avg_var = bias_variance_decomp( tree, X_train, y_train, X_test, y_test, loss='mse', random_seed=123) print('Average expected loss: %.3f' % avg_expected_loss) print('Average bias: %.3f' % avg_bias) print('Average variance: %.3f' % avg_var) Average expected loss: 31.536 Average bias: 14.096 Average variance: 17.440 For comparison, the bias-variance decomposition of a bagging regressor is shown below, which should intuitively have a lower variance than a single decision tree: from sklearn.ensemble import BaggingRegressor tree = DecisionTreeRegressor(random_state=123) bag = BaggingRegressor(base_estimator=tree, n_estimators=100, random_state=123) avg_expected_loss, avg_bias, avg_var = bias_variance_decomp( bag, X_train, y_train, X_test, y_test, loss='mse', random_seed=123) print('Average expected loss: %.3f' % avg_expected_loss) print('Average bias: %.3f' % avg_bias) print('Average variance: %.3f' % avg_var) Average expected loss: 18.620 Average bias: 15.461 Average variance: 3.159 Example 3 -- TensorFlow/Keras Support Since mlxtend v0.18.0, the bias_variance_decomp now supports Keras models. Note that the original model is reset in each round (before refitting it to the bootstrap samples). from mlxtend.evaluate import bias_variance_decomp from mlxtend.data import boston_housing_data from sklearn.model_selection import train_test_split from sklearn.metrics import mean_squared_error import tensorflow as tf import numpy as np np.random.seed(1) tf.random.set_seed(1) X, y = boston_housing_data() X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=123, shuffle=True) model = tf.keras.Sequential([ tf.keras.layers.Dense(32, activation=tf.nn.relu), tf.keras.layers.Dense(1) ]) optimizer = tf.keras.optimizers.Adam() model.compile(loss='mean_squared_error', optimizer=optimizer) model.fit(X_train, y_train, epochs=100, verbose=0) mean_squared_error(model.predict(X_test), y_test) 32.69300595184836 Note that it is highly recommended to use the same number of training epochs that you would use on the original training set to ensure convergence: np.random.seed(1) tf.random.set_seed(1) avg_expected_loss, avg_bias, avg_var = bias_variance_decomp( model, X_train, y_train, X_test, y_test, loss='mse', num_rounds=100, random_seed=123, epochs=200, # fit_param verbose=0) # fit_param print('Average expected loss: %.3f' % avg_expected_loss) print('Average bias: %.3f' % avg_bias) print('Average variance: %.3f' % avg_var) Average expected loss: 32.740 Average bias: 27.474 Average variance: 5.265 API bias_variance_decomp(estimator, X_train, y_train, X_test, y_test, loss='0-1_loss', num_rounds=200, random_seed=None, fit_params) estimator : object A classifier or regressor object or class implementing both a fit and predict method similar to the scikit-learn API. X_train : array-like, shape=(num_examples, num_features) A training dataset for drawing the bootstrap samples to carry out the bias-variance decomposition. y_train : array-like, shape=(num_examples) Targets (class labels, continuous values in case of regression) associated with the X_train examples. X_test : array-like, shape=(num_examples, num_features) The test dataset for computing the average loss, bias, and variance. y_test : array-like, shape=(num_examples) Targets (class labels, continuous values in case of regression) associated with the X_test examples. loss : str (default='0-1_loss') Loss function for performing the bias-variance decomposition. Currently allowed values are '0-1_loss' and 'mse'. num_rounds : int (default=200) Number of bootstrap rounds (sampling from the training set) for performing the bias-variance decomposition. Each bootstrap sample has the same size as the original training set. random_seed : int (default=None) Random seed for the bootstrap sampling used for the bias-variance decomposition. fit_params : additional parameters Additional parameters to be passed to the .fit() function of the estimator when it is fit to the bootstrap samples. Returns avg_expected_loss, avg_bias, avg_var : returns the average expected average bias, and average bias (all floats), where the average is computed over the data points in the test set. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/bias_variance_decomp/","title":"bias_variance_decomp: Bias-variance decomposition for classification and regression losses"},{"location":"user_guide/evaluate/bias_variance_decomp/#bias_variance_decomp-bias-variance-decomposition-for-classification-and-regression-losses","text":"Bias variance decomposition of machine learning algorithms for various loss functions. from mlxtend.evaluate import bias_variance_decomp","title":"bias_variance_decomp: Bias-variance decomposition for classification and regression losses"},{"location":"user_guide/evaluate/bias_variance_decomp/#overview","text":"Often, researchers use the terms bias and variance or \"bias-variance tradeoff\" to describe the performance of a model -- i.e., you may stumble upon talks, books, or articles where people say that a model has a high variance or high bias. So, what does that mean? In general, we might say that \"high variance\" is proportional to overfitting, and \"high bias\" is proportional to underfitting. Anyways, why are we attempting to do this bias-variance decomposition in the first place? The decomposition of the loss into bias and variance helps us understand learning algorithms, as these concepts are correlated to underfitting and overfitting. To use the more formal terms for bias and variance, assume we have a point estimator \\hat{\\theta} of some parameter or function \\theta . Then, the bias is commonly defined as the difference between the expected value of the estimator and the parameter that we want to estimate: \\text{Bias} = E[\\hat{\\theta}] - \\theta. If the bias is larger than zero, we also say that the estimator is positively biased, if the bias is smaller than zero, the estimator is negatively biased, and if the bias is exactly zero, the estimator is unbiased. Similarly, we define the variance as the difference between the expected value of the squared estimator minus the squared expectation of the estimator: \\text{Var}(\\hat{\\theta}) = E\\big[\\hat{\\theta}^2\\big] - \\bigg(E\\big[\\hat{\\theta}\\big]\\bigg)^2. Note that in the context of this lecture, it will be more convenient to write the variance in its alternative form: \\text{Var}(\\hat{\\theta}) = E[(E[{\\hat{\\theta}}] - \\hat{\\theta})^2]. To illustrate the concept further in context of machine learning ... Suppose there is an unknown target function or \"true function\" to which we do want to approximate. Now, suppose we have different training sets drawn from an unknown distribution defined as \"true function + noise.\" The following plot shows different linear regression models, each fit to a different training set. None of these hypotheses approximate the true function well, except at two points (around x=-10 and x=6). Here, we can say that the bias is large because the difference between the true value and the predicted value, on average (here, average means \"expectation of the training sets\" not \"expectation over examples in the training set\"), is large: The next plot shows different unpruned decision tree models, each fit to a different training set. Note that these hypotheses fit the training data very closely. However, if we would consider the expectation over training sets, the average hypothesis would fit the true function perfectly (given that the noise is unbiased and has an expected value of 0). As we can see, the variance is very large, since on average, a prediction differs a lot from the expectation value of the prediction:","title":"Overview"},{"location":"user_guide/evaluate/bias_variance_decomp/#bias-variance-decomposition-of-the-squared-loss","text":"We can decompose a loss function such as the squared loss into three terms, a variance, bias, and a noise term (and the same is true for the decomposition of the 0-1 loss later). However, for simplicity, we will ignore the noise term. Before we introduce the bias-variance decomposition of the 0-1 loss for classification, let us start with the decomposition of the squared loss as an easy warm-up exercise to get familiar with the overall concept. The previous section already listed the common formal definitions of bias and variance, however, let us define them again for convenience: \\text{Bias}(\\hat{\\theta}) = E[\\hat{\\theta}] - \\theta, \\quad \\text{Var}(\\hat{\\theta}) = E[(E[{\\hat{\\theta}}] - \\hat{\\theta})^2]. Recall that in the context of these machine learning lecture (notes), we defined the true or target function as y = f(x) , the predicted target value as \\hat{y} = \\hat{f}(x) = h(x) , and the squared loss as S = (y - \\hat{y})^2 . (I use S here because it will be easier to tell it apart from the E , which we use for the expectation in this lecture.) Note that unless noted otherwise, the expectation is over training sets! To get started with the squared error loss decomposition into bias and variance, let use do some algebraic manipulation, i.e., adding and subtracting the expected value of \\hat{y} and then expanding the expression using the quadratic formula (a+b)^2 = a^2 + b^2 + 2ab) : \\begin{equation} \\begin{split} S = (y - \\hat{y})^2 \\\\ (y - \\hat{y})^2 &= (y - E[{\\hat{y}}] + E[{\\hat{y}}] - \\hat{y})^2 \\\\ &= (y-E[{\\hat{y}}])^2 + (E[{\\hat{y}}] - y)^2 + 2(y - E[\\hat{y}])(E[\\hat{y}] - \\hat{y}). \\end{split} \\end{equation} Next, we just use the expectation on both sides, and we are already done: \\begin{align} E[S] &= E[(y - \\hat{y})^2] \\\\ E[(y - \\hat{y})^2] &= (y-E[{\\hat{y}}])^2 + E[(E[{\\hat{y}}] - \\hat{y})^2]\\\\ &= \\text{[Bias]}^2 + \\text{Variance}. \\end{align} You may wonder what happened to the \" 2ab \" term ( 2(y - E[\\hat{y}])(E[\\hat{y}] - \\hat{y}) ) when we used the expectation. It turns that it evaluates to zero and hence vanishes from the equation, which can be shown as follows: \\begin{align} E[2(y - E[{\\hat{y}}])(E[{\\hat{y}}] - \\hat{y})] &= 2 E[(y - E[{\\hat{y}}])(E[{\\hat{y}}] - \\hat{y})] \\\\ &= 2(y - E[{\\hat{y}}])E[(E[{\\hat{y}}] - \\hat{y})] \\\\ &= 2(y - E[{\\hat{y}}])(E[E[{\\hat{y}}]] - E[\\hat{y}])\\\\ &= 2(y - E[{\\hat{y}}])(E[{\\hat{y}}] - E[{\\hat{y}}]) \\\\ &= 0. \\end{align} So, this is the canonical decomposition of the squared error loss into bias and variance. The next section will discuss some approaches that have been made to decompose the 0-1 loss that we commonly use for classification accuracy or error. The following figure is a sketch of variance and bias in relation to the training error and generalization error -- how high variance related to overfitting, and how large bias relates to underfitting:","title":"Bias-Variance Decomposition of the Squared Loss"},{"location":"user_guide/evaluate/bias_variance_decomp/#bias-variance-decomposition-of-the-0-1-loss","text":"Note that decomposing the 0-1 loss into bias and variance components is not as straight-forward as for the squared error loss. To quote Pedro Domingos, a well-known machine learning researcher and professor at University of Washington: \"several authors have proposed bias-variance decompositions related to zero-one loss (Kong & Dietterich, 1995; Breiman, 1996b; Kohavi & Wolpert, 1996; Tibshirani, 1996; Friedman, 1997). However, each of these decompositions has significant shortcomings.\". [1] In fact, the paper this quote was taken from may offer the most intuitive and general formulation at this point. However, we will first, for simplicity, go over Kong & Dietterich formulation [2] of the 0-1 loss decomposition, which is the same as Domingos's but excluding the noise term (for simplicity). The table below summarizes the relevant terms we used for the squared loss in relation to the 0-1 loss. Recall that the 0-1 loss, L , is 0 if a class label is predicted correctly, and one otherwise. The main prediction for the squared error loss is simply the average over the predictions E[\\hat{y}] (the expectation is over training sets), for the 0-1 loss Kong & Dietterich and Domingos defined it as the mode. I.e., if a model predicts the label one more than 50% of the time (considering all possible training sets), then the main prediction is 1, and 0 otherwise. - Squared Loss 0-1 Loss Single loss (y - \\hat{y})^2 L(y, \\hat{y}) Expected loss E[(y - \\hat{y})^2] E[L(y, \\hat{y})] Main prediction E[\\hat{y}] mean (average) mode Bias ^2 (y-E[{\\hat{y}}])^2 L(y, E[\\hat{y}]) Variance E[(E[{\\hat{y}}] - \\hat{y})^2] E[L(\\hat{y}, E[\\hat{y}])] Hence, as result from using the mode to define the main prediction of the 0-1 loss, the bias is 1 if the main prediction does not agree with the true label y , and 0 otherwise: Bias = \\begin{cases} 1 \\text{ if } y \\neq E[{\\hat{y}}], \\\\ 0 \\text{ otherwise}. \\end{cases} The variance of the 0-1 loss is defined as the probability that the predicted label does not match the main prediction: Variance = P(\\hat{y} \\neq E[\\hat{{y}}]). Next, let us take a look at what happens to the loss if the bias is 0. Given the general definition of the loss, loss = bias + variance, if the bias is 0, then we define the loss as the variance: Loss = 0 + Variance = Loss = P(\\hat{y} \\neq y) = Variance = P(\\hat{y} \\neq E[\\hat{{y}}]). In other words, if a model has zero bias, it's loss is entirely defined by the variance, which is intuitive if we think of variance in the context of being proportional overfitting. The more surprising scenario is if the bias is equal to 1. If the bias is equal to 1, as explained by Pedro Domingos, the increasing the variance can decrease the loss, which is an interesting observation. This can be seen by first rewriting the 0-1 loss function as Loss = P(\\hat{y} \\neq y) = 1 - P(\\hat{y} = y). (Note that we have not done anything new, yet.) Now, if we look at the previous equation of the bias, if the bias is 1, we have y \\neq E[{\\hat{y}}] . If y is not equal to the main prediction, but y is also is equal to \\hat{y} , then \\hat{y} must be equal to the main prediction. Using the \"inverse\" (\"1 minus\"), we can then write the loss as Loss = P(\\hat{y} \\neq y) = 1 - P(\\hat{y} = y) = 1 - P(\\hat{y} \\neq E[{\\hat{y}}]). Since the bias is 1, the loss is hence defined as \"loss = bias - variance\" if the bias is 1 (or \"loss = 1 - variance\"). This might be quite unintuitive at first, but the explanations Kong, Dietterich, and Domingos offer was that if a model has a very high bias such that it main prediction is always wrong, increasing the variance can be beneficial, since increasing the variance would push the decision boundary, which might lead to some correct predictions just by chance then. In other words, for scenarios with high bias, increasing the variance can improve (decrease) the loss!","title":"Bias-Variance Decomposition of the 0-1 Loss"},{"location":"user_guide/evaluate/bias_variance_decomp/#references","text":"[1] Domingos, Pedro. \"A unified bias-variance decomposition.\" Proceedings of 17th International Conference on Machine Learning. 2000. [2] Dietterich, Thomas G., and Eun Bae Kong. Machine learning bias, statistical bias, and statistical variance of decision tree algorithms. Technical report, Department of Computer Science, Oregon State University, 1995.","title":"References"},{"location":"user_guide/evaluate/bias_variance_decomp/#example-1-bias-variance-decomposition-of-a-decision-tree-classifier","text":"from mlxtend.evaluate import bias_variance_decomp from sklearn.tree import DecisionTreeClassifier from mlxtend.data import iris_data from sklearn.model_selection import train_test_split X, y = iris_data() X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=123, shuffle=True, stratify=y) tree = DecisionTreeClassifier(random_state=123) avg_expected_loss, avg_bias, avg_var = bias_variance_decomp( tree, X_train, y_train, X_test, y_test, loss='0-1_loss', random_seed=123) print('Average expected loss: %.3f' % avg_expected_loss) print('Average bias: %.3f' % avg_bias) print('Average variance: %.3f' % avg_var) Average expected loss: 0.062 Average bias: 0.022 Average variance: 0.040 For comparison, the bias-variance decomposition of a bagging classifier, which should intuitively have a lower variance compared than a single decision tree: from sklearn.ensemble import BaggingClassifier tree = DecisionTreeClassifier(random_state=123) bag = BaggingClassifier(base_estimator=tree, n_estimators=100, random_state=123) avg_expected_loss, avg_bias, avg_var = bias_variance_decomp( bag, X_train, y_train, X_test, y_test, loss='0-1_loss', random_seed=123) print('Average expected loss: %.3f' % avg_expected_loss) print('Average bias: %.3f' % avg_bias) print('Average variance: %.3f' % avg_var) Average expected loss: 0.048 Average bias: 0.022 Average variance: 0.026","title":"Example 1 -- Bias Variance Decomposition of a Decision Tree Classifier"},{"location":"user_guide/evaluate/bias_variance_decomp/#example-2-bias-variance-decomposition-of-a-decision-tree-regressor","text":"from mlxtend.evaluate import bias_variance_decomp from sklearn.tree import DecisionTreeRegressor from mlxtend.data import boston_housing_data from sklearn.model_selection import train_test_split X, y = boston_housing_data() X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=123, shuffle=True) tree = DecisionTreeRegressor(random_state=123) avg_expected_loss, avg_bias, avg_var = bias_variance_decomp( tree, X_train, y_train, X_test, y_test, loss='mse', random_seed=123) print('Average expected loss: %.3f' % avg_expected_loss) print('Average bias: %.3f' % avg_bias) print('Average variance: %.3f' % avg_var) Average expected loss: 31.536 Average bias: 14.096 Average variance: 17.440 For comparison, the bias-variance decomposition of a bagging regressor is shown below, which should intuitively have a lower variance than a single decision tree: from sklearn.ensemble import BaggingRegressor tree = DecisionTreeRegressor(random_state=123) bag = BaggingRegressor(base_estimator=tree, n_estimators=100, random_state=123) avg_expected_loss, avg_bias, avg_var = bias_variance_decomp( bag, X_train, y_train, X_test, y_test, loss='mse', random_seed=123) print('Average expected loss: %.3f' % avg_expected_loss) print('Average bias: %.3f' % avg_bias) print('Average variance: %.3f' % avg_var) Average expected loss: 18.620 Average bias: 15.461 Average variance: 3.159","title":"Example 2 -- Bias Variance Decomposition of a Decision Tree Regressor"},{"location":"user_guide/evaluate/bias_variance_decomp/#example-3-tensorflowkeras-support","text":"Since mlxtend v0.18.0, the bias_variance_decomp now supports Keras models. Note that the original model is reset in each round (before refitting it to the bootstrap samples). from mlxtend.evaluate import bias_variance_decomp from mlxtend.data import boston_housing_data from sklearn.model_selection import train_test_split from sklearn.metrics import mean_squared_error import tensorflow as tf import numpy as np np.random.seed(1) tf.random.set_seed(1) X, y = boston_housing_data() X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=123, shuffle=True) model = tf.keras.Sequential([ tf.keras.layers.Dense(32, activation=tf.nn.relu), tf.keras.layers.Dense(1) ]) optimizer = tf.keras.optimizers.Adam() model.compile(loss='mean_squared_error', optimizer=optimizer) model.fit(X_train, y_train, epochs=100, verbose=0) mean_squared_error(model.predict(X_test), y_test) 32.69300595184836 Note that it is highly recommended to use the same number of training epochs that you would use on the original training set to ensure convergence: np.random.seed(1) tf.random.set_seed(1) avg_expected_loss, avg_bias, avg_var = bias_variance_decomp( model, X_train, y_train, X_test, y_test, loss='mse', num_rounds=100, random_seed=123, epochs=200, # fit_param verbose=0) # fit_param print('Average expected loss: %.3f' % avg_expected_loss) print('Average bias: %.3f' % avg_bias) print('Average variance: %.3f' % avg_var) Average expected loss: 32.740 Average bias: 27.474 Average variance: 5.265","title":"Example 3 -- TensorFlow/Keras Support"},{"location":"user_guide/evaluate/bias_variance_decomp/#api","text":"bias_variance_decomp(estimator, X_train, y_train, X_test, y_test, loss='0-1_loss', num_rounds=200, random_seed=None, fit_params) estimator : object A classifier or regressor object or class implementing both a fit and predict method similar to the scikit-learn API. X_train : array-like, shape=(num_examples, num_features) A training dataset for drawing the bootstrap samples to carry out the bias-variance decomposition. y_train : array-like, shape=(num_examples) Targets (class labels, continuous values in case of regression) associated with the X_train examples. X_test : array-like, shape=(num_examples, num_features) The test dataset for computing the average loss, bias, and variance. y_test : array-like, shape=(num_examples) Targets (class labels, continuous values in case of regression) associated with the X_test examples. loss : str (default='0-1_loss') Loss function for performing the bias-variance decomposition. Currently allowed values are '0-1_loss' and 'mse'. num_rounds : int (default=200) Number of bootstrap rounds (sampling from the training set) for performing the bias-variance decomposition. Each bootstrap sample has the same size as the original training set. random_seed : int (default=None) Random seed for the bootstrap sampling used for the bias-variance decomposition. fit_params : additional parameters Additional parameters to be passed to the .fit() function of the estimator when it is fit to the bootstrap samples. Returns avg_expected_loss, avg_bias, avg_var : returns the average expected average bias, and average bias (all floats), where the average is computed over the data points in the test set. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/bias_variance_decomp/","title":"API"},{"location":"user_guide/evaluate/bootstrap/","text":"bootstrap: The ordinary nonparametric boostrap for arbitrary parameters An implementation of the ordinary nonparametric bootstrap to bootstrap a single statistic (for example, the mean. median, R^2 of a regression fit, and so forth). from mlxtend.evaluate import bootstrap Overview The bootstrap offers an easy and effective way to estimate the distribution of a statistic via simulation, by drawing (or generating) new samples from an existing sample with replacement. Note that the bootstrap does not require making any assumptions about the sample statistic or dataset being normally distributed. Using the bootstrap, we can estimate sample statistics and compute the standard error of the mean and confidence intervals as if we have drawn a number of samples from an infinite population. In a nutshell, the bootstrap procedure can be described as follows: Draw a sample with replacement Compute the sample statistic Repeat step 1-2 n times Compute the standard deviation (standard error of the mean of the statistic) Compute the confidence interval Or, in simple terms, we can interpret the bootstrap a means of drawing a potentially endless number of (new) samples from a population by resampling the original dataset. Note that the term \"bootstrap replicate\" is being used quite loosely in current literature; many researchers and practitioners use it to define the number of bootstrap samples we draw from the original dataset. However, in the context of this documentation and the code annotation, we use the original definition of bootstrap repliactes and use it to refer to the statistic computed from a bootstrap sample. References [1] Efron, Bradley, and Robert J. Tibshirani. An introduction to the bootstrap. CRC press, 1994. Management of Data (ACM SIGMOD '97), pages 265-276, 1997. Example 1 -- Bootstrapping the Mean This simple example illustrates how you could bootstrap the mean of a sample. import numpy as np from mlxtend.evaluate import bootstrap rng = np.random.RandomState(123) x = rng.normal(loc=5., size=100) original, std_err, ci_bounds = bootstrap(x, num_rounds=1000, func=np.mean, ci=0.95, seed=123) print('Mean: %.2f, SE: +/- %.2f, CI95: [%.2f, %.2f]' % (original, std_err, ci_bounds[0], ci_bounds[1])) Mean: 5.03, SE: +/- 0.11, CI95: [4.80, 5.26] Example 2 - Bootstrapping a Regression Fit This example illustrates how you can bootstrap the R^2 of a regression fit on the training data. from mlxtend.data import autompg_data from sklearn.linear_model import LinearRegression from sklearn.metrics import r2_score X, y = autompg_data() lr = LinearRegression() def r2_fit(X, model=lr): x, y = X[:, 0].reshape(-1, 1), X[:, 1] pred = lr.fit(x, y).predict(x) return r2_score(y, pred) original, std_err, ci_bounds = bootstrap(X, num_rounds=1000, func=r2_fit, ci=0.95, seed=123) print('Mean: %.2f, SE: +/- %.2f, CI95: [%.2f, %.2f]' % (original, std_err, ci_bounds[0], ci_bounds[1])) Mean: 0.90, SE: +/- 0.01, CI95: [0.89, 0.92] API bootstrap(x, func, num_rounds=1000, ci=0.95, ddof=1, seed=None) Implements the ordinary nonparametric bootstrap Parameters x : NumPy array, shape=(n_samples, [n_columns]) An one or multidimensional array of data records func : A function which computes a statistic that is used to compute the bootstrap replicates (the statistic computed from the bootstrap samples). This function must return a scalar value. For example, np.mean or np.median would be an acceptable argument for func if x is a 1-dimensional array or vector. num_rounds : int (default=1000) The number of bootstrap samnples to draw where each bootstrap sample has the same number of records as the original dataset. ci : int (default=0.95) An integer in the range (0, 1) that represents the confidence level for computing the confidence interval. For example, ci=0.95 (default) will compute the 95% confidence interval from the bootstrap replicates. ddof : int The delta degrees of freedom used when computing the standard error. seed : int or None (default=None) Random seed for generating bootstrap samples. Returns original, standard_error, (lower_ci, upper_ci) : tuple Returns the statistic of the original sample ( original ), the standard error of the estimate, and the respective confidence interval bounds. Examples >>> from mlxtend.evaluate import bootstrap >>> rng = np.random.RandomState(123) >>> x = rng.normal(loc=5., size=100) >>> original, std_err, ci_bounds = bootstrap(x, ... num_rounds=1000, ... func=np.mean, ... ci=0.95, ... seed=123) >>> print('Mean: %.2f, SE: +/- %.2f, CI95: [%.2f, %.2f]' % (original, ... std_err, ... ci_bounds[0], ... ci_bounds[1])) Mean: 5.03, SE: +/- 0.11, CI95: [4.80, 5.26] >>> For more usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/bootstrap/","title":"bootstrap: The ordinary nonparametric boostrap for arbitrary parameters"},{"location":"user_guide/evaluate/bootstrap/#bootstrap-the-ordinary-nonparametric-boostrap-for-arbitrary-parameters","text":"An implementation of the ordinary nonparametric bootstrap to bootstrap a single statistic (for example, the mean. median, R^2 of a regression fit, and so forth). from mlxtend.evaluate import bootstrap","title":"bootstrap: The ordinary nonparametric boostrap for arbitrary parameters"},{"location":"user_guide/evaluate/bootstrap/#overview","text":"The bootstrap offers an easy and effective way to estimate the distribution of a statistic via simulation, by drawing (or generating) new samples from an existing sample with replacement. Note that the bootstrap does not require making any assumptions about the sample statistic or dataset being normally distributed. Using the bootstrap, we can estimate sample statistics and compute the standard error of the mean and confidence intervals as if we have drawn a number of samples from an infinite population. In a nutshell, the bootstrap procedure can be described as follows: Draw a sample with replacement Compute the sample statistic Repeat step 1-2 n times Compute the standard deviation (standard error of the mean of the statistic) Compute the confidence interval Or, in simple terms, we can interpret the bootstrap a means of drawing a potentially endless number of (new) samples from a population by resampling the original dataset. Note that the term \"bootstrap replicate\" is being used quite loosely in current literature; many researchers and practitioners use it to define the number of bootstrap samples we draw from the original dataset. However, in the context of this documentation and the code annotation, we use the original definition of bootstrap repliactes and use it to refer to the statistic computed from a bootstrap sample.","title":"Overview"},{"location":"user_guide/evaluate/bootstrap/#references","text":"[1] Efron, Bradley, and Robert J. Tibshirani. An introduction to the bootstrap. CRC press, 1994. Management of Data (ACM SIGMOD '97), pages 265-276, 1997.","title":"References"},{"location":"user_guide/evaluate/bootstrap/#example-1-bootstrapping-the-mean","text":"This simple example illustrates how you could bootstrap the mean of a sample. import numpy as np from mlxtend.evaluate import bootstrap rng = np.random.RandomState(123) x = rng.normal(loc=5., size=100) original, std_err, ci_bounds = bootstrap(x, num_rounds=1000, func=np.mean, ci=0.95, seed=123) print('Mean: %.2f, SE: +/- %.2f, CI95: [%.2f, %.2f]' % (original, std_err, ci_bounds[0], ci_bounds[1])) Mean: 5.03, SE: +/- 0.11, CI95: [4.80, 5.26]","title":"Example 1 -- Bootstrapping the Mean"},{"location":"user_guide/evaluate/bootstrap/#example-2-bootstrapping-a-regression-fit","text":"This example illustrates how you can bootstrap the R^2 of a regression fit on the training data. from mlxtend.data import autompg_data from sklearn.linear_model import LinearRegression from sklearn.metrics import r2_score X, y = autompg_data() lr = LinearRegression() def r2_fit(X, model=lr): x, y = X[:, 0].reshape(-1, 1), X[:, 1] pred = lr.fit(x, y).predict(x) return r2_score(y, pred) original, std_err, ci_bounds = bootstrap(X, num_rounds=1000, func=r2_fit, ci=0.95, seed=123) print('Mean: %.2f, SE: +/- %.2f, CI95: [%.2f, %.2f]' % (original, std_err, ci_bounds[0], ci_bounds[1])) Mean: 0.90, SE: +/- 0.01, CI95: [0.89, 0.92]","title":"Example 2 - Bootstrapping a Regression Fit"},{"location":"user_guide/evaluate/bootstrap/#api","text":"bootstrap(x, func, num_rounds=1000, ci=0.95, ddof=1, seed=None) Implements the ordinary nonparametric bootstrap Parameters x : NumPy array, shape=(n_samples, [n_columns]) An one or multidimensional array of data records func : A function which computes a statistic that is used to compute the bootstrap replicates (the statistic computed from the bootstrap samples). This function must return a scalar value. For example, np.mean or np.median would be an acceptable argument for func if x is a 1-dimensional array or vector. num_rounds : int (default=1000) The number of bootstrap samnples to draw where each bootstrap sample has the same number of records as the original dataset. ci : int (default=0.95) An integer in the range (0, 1) that represents the confidence level for computing the confidence interval. For example, ci=0.95 (default) will compute the 95% confidence interval from the bootstrap replicates. ddof : int The delta degrees of freedom used when computing the standard error. seed : int or None (default=None) Random seed for generating bootstrap samples. Returns original, standard_error, (lower_ci, upper_ci) : tuple Returns the statistic of the original sample ( original ), the standard error of the estimate, and the respective confidence interval bounds. Examples >>> from mlxtend.evaluate import bootstrap >>> rng = np.random.RandomState(123) >>> x = rng.normal(loc=5., size=100) >>> original, std_err, ci_bounds = bootstrap(x, ... num_rounds=1000, ... func=np.mean, ... ci=0.95, ... seed=123) >>> print('Mean: %.2f, SE: +/- %.2f, CI95: [%.2f, %.2f]' % (original, ... std_err, ... ci_bounds[0], ... ci_bounds[1])) Mean: 5.03, SE: +/- 0.11, CI95: [4.80, 5.26] >>> For more usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/bootstrap/","title":"API"},{"location":"user_guide/evaluate/bootstrap_point632_score/","text":"bootstrap_point632_score: The .632 and .632+ boostrap for classifier evaluation An implementation of the .632 bootstrap to evaluate supervised learning algorithms. from mlxtend.evaluate import bootstrap_point632_score Overview Originally, the bootstrap method aims to determine the statistical properties of an estimator when the underlying distribution was unknown and additional samples are not available. Now, in order to exploit this method for the evaluation of predictive models, such as hypotheses for classification and regression, we may prefer a slightly different approach to bootstrapping using the so-called Out-Of-Bag (OOB) or Leave-One-Out Bootstrap (LOOB) technique. Here, we use out-of-bag samples as test sets for evaluation instead of evaluating the model on the training data. Out-of-bag samples are the unique sets of instances that are not used for model fitting as shown in the figure below [1]. The figure above illustrates how three random bootstrap samples drawn from an exemplary ten-sample dataset ( X_1,X_2, ..., X_{10} ) and their out-of-bag sample for testing may look like. In practice, Bradley Efron and Robert Tibshirani recommend drawing 50 to 200 bootstrap samples as being sufficient for reliable estimates [2]. .632 Bootstrap In 1983, Bradley Efron described the .632 Estimate , a further improvement to address the pessimistic bias of the bootstrap cross-validation approach described above [3]. The pessimistic bias in the \"classic\" bootstrap method can be attributed to the fact that the bootstrap samples only contain approximately 63.2% of the unique samples from the original dataset. For instance, we can compute the probability that a given sample from a dataset of size n is not drawn as a bootstrap sample as P (\\text{not chosen}) = \\bigg(1 - \\frac{1}{n}\\bigg)^n, which is asymptotically equivalent to \\frac{1}{e} \\approx 0.368 as n \\rightarrow \\infty. Vice versa, we can then compute the probability that a sample is chosen as P (\\text{chosen}) = 1 - \\bigg(1 - \\frac{1}{n}\\bigg)^n \\approx 0.632 for reasonably large datasets, so that we'd select approximately 0.632 \\times n uniques samples as bootstrap training sets and reserve 0.368 \\times n out-of-bag samples for testing in each iteration. Now, to address the bias that is due to this the sampling with replacement, Bradley Efron proposed the .632 Estimate that we mentioned earlier, which is computed via the following equation: \\text{ACC}_{boot} = \\frac{1}{b} \\sum_{i=1}^b \\big(0.632 \\cdot \\text{ACC}_{h, i} + 0.368 \\cdot \\text{ACC}_{train}\\big), where \\text{ACC}_{train} is the accuracy computed on the whole training set, and \\text{ACC}_{h, i} is the accuracy on the out-of-bag sample. .632+ Bootstrap Now, while the .632 Boostrap attempts to address the pessimistic bias of the estimate, an optimistic bias may occur with models that tend to overfit so that Bradley Efron and Robert Tibshirani proposed the The .632+ Bootstrap Method (Efron and Tibshirani, 1997). Instead of using a fixed \"weight\" \\omega = 0.632 in ACC_{\\text{boot}} = \\frac{1}{b} \\sum_{i=1}^b \\big(\\omega \\cdot \\text{ACC}_{h, i} + (1-\\omega) \\cdot \\text{ACC}_{train} \\big), we compute the weight \\gamma as \\omega = \\frac{0.632}{1 - 0.368 \\times R}, where R is the relative overfitting rate R = \\frac{(-1) \\times (\\text{ACC}_{h, i} - \\text{ACC}_{train})}{\\gamma - (1 -\\text{ACC}_{h, i})}. (Since we are plugging \\omega into the equation for computing ACC_{boot} that we defined above, \\text{ACC}_{h, i} and \\text{ACC}_{train} still refer to the out-of-bag accuracy in the i th bootstrap round and the whole training set accuracy, respectively.) Further, we need to determine the no-information rate \\gamma in order to compute R . For instance, we can compute \\gamma by fitting a model to a dataset that contains all possible combinations between samples x_{i'} and target class labels y_{i} \u2014 we pretend that the observations and class labels are independent: \\gamma = \\frac{1}{n^2} \\sum_{i=1}^{n} \\sum_{i '=1}^{n} L(y_{i}, f(x_{i '})). Alternatively, we can estimate the no-information rate \\gamma as follows: \\gamma = \\sum_{k=1}^K p_k (1 - q_k), where p_k is the proportion of class k samples observed in the dataset, and q_k is the proportion of class k samples that the classifier predicts in the dataset. References [1] https://sebastianraschka.com/blog/2016/model-evaluation-selection-part2.html [2] Efron, Bradley, and Robert J. Tibshirani. An introduction to the bootstrap. CRC press, 1994. Management of Data (ACM SIGMOD '97), pages 265-276, 1997. [3] Efron, Bradley. 1983. \u201cEstimating the Error Rate of a Prediction Rule: Improvement on Cross-Validation.\u201d Journal of the American Statistical Association 78 (382): 316. doi:10.2307/2288636. [4] Efron, Bradley, and Robert Tibshirani. 1997. \u201cImprovements on Cross-Validation: The .632+ Bootstrap Method.\u201d Journal of the American Statistical Association 92 (438): 548. doi:10.2307/2965703. Example 1 -- Evaluating the predictive performance of a model via the classic out-of-bag Bootstrap The bootstrap_point632_score function mimics the behavior of scikit-learn's `cross_val_score, and a typically usage example is shown below: from sklearn import datasets from sklearn.tree import DecisionTreeClassifier from mlxtend.evaluate import bootstrap_point632_score import numpy as np iris = datasets.load_iris() X = iris.data y = iris.target tree = DecisionTreeClassifier(random_state=123) # Model accuracy scores = bootstrap_point632_score(tree, X, y, method='oob') acc = np.mean(scores) print('Accuracy: %.2f%%' % (100*acc)) # Confidence interval lower = np.percentile(scores, 2.5) upper = np.percentile(scores, 97.5) print('95%% Confidence interval: [%.2f, %.2f]' % (100*lower, 100*upper)) Accuracy: 94.45% 95% Confidence interval: [87.71, 100.00] Example 2 -- Evaluating the predictive performance of a model via the .632 Bootstrap from sklearn import datasets from sklearn.tree import DecisionTreeClassifier from mlxtend.evaluate import bootstrap_point632_score import numpy as np iris = datasets.load_iris() X = iris.data y = iris.target tree = DecisionTreeClassifier(random_state=123) # Model accuracy scores = bootstrap_point632_score(tree, X, y) acc = np.mean(scores) print('Accuracy: %.2f%%' % (100*acc)) # Confidence interval lower = np.percentile(scores, 2.5) upper = np.percentile(scores, 97.5) print('95%% Confidence interval: [%.2f, %.2f]' % (100*lower, 100*upper)) Accuracy: 96.42% 95% Confidence interval: [92.41, 100.00] Example 3 -- Evaluating the predictive performance of a model via the .632+ Bootstrap from sklearn import datasets from sklearn.tree import DecisionTreeClassifier from mlxtend.evaluate import bootstrap_point632_score import numpy as np iris = datasets.load_iris() X = iris.data y = iris.target tree = DecisionTreeClassifier(random_state=123) # Model accuracy scores = bootstrap_point632_score(tree, X, y, method='.632+') acc = np.mean(scores) print('Accuracy: %.2f%%' % (100*acc)) # Confidence interval lower = np.percentile(scores, 2.5) upper = np.percentile(scores, 97.5) print('95%% Confidence interval: [%.2f, %.2f]' % (100*lower, 100*upper)) Accuracy: 96.29% 95% Confidence interval: [91.86, 98.92] API bootstrap_point632_score(estimator, X, y, n_splits=200, method='.632', scoring_func=None, predict_proba=False, random_seed=None, clone_estimator=True) Implementation of the .632 [1] and .632+ [2] bootstrap for supervised learning References: - [1] Efron, Bradley. 1983. \"Estimating the Error Rate of a Prediction Rule: Improvement on Cross-Validation.\" Journal of the American Statistical Association 78 (382): 316. doi:10.2307/2288636. - [2] Efron, Bradley, and Robert Tibshirani. 1997. \"Improvements on Cross-Validation: The .632+ Bootstrap Method.\" Journal of the American Statistical Association 92 (438): 548. doi:10.2307/2965703. Parameters estimator : object An estimator for classification or regression that follows the scikit-learn API and implements \"fit\" and \"predict\" methods. X : array-like The data to fit. Can be, for example a list, or an array at least 2d. y : array-like, optional, default: None The target variable to try to predict in the case of supervised learning. n_splits : int (default=200) Number of bootstrap iterations. Must be larger than 1. method : str (default='.632') The bootstrap method, which can be either - 1) '.632' bootstrap (default) - 2) '.632+' bootstrap - 3) 'oob' (regular out-of-bag, no weighting) for comparison studies. scoring_func : callable, Score function (or loss function) with signature scoring_func(y, y_pred, **kwargs) . If none, uses classification accuracy if the estimator is a classifier and mean squared error if the estimator is a regressor. predict_proba : bool Whether to use the predict_proba function for the estimator argument. This is to be used in conjunction with scoring_func which takes in probability values instead of actual predictions. For example, if the scoring_func is :meth: sklearn.metrics.roc_auc_score , then use predict_proba=True . Note that this requires estimator to have predict_proba method implemented. random_seed : int (default=None) If int, random_seed is the seed used by the random number generator. clone_estimator : bool (default=True) Clones the estimator if true, otherwise fits the original. Returns scores : array of float, shape=(len(list(n_splits)),) Array of scores of the estimator for each bootstrap replicate. Examples >>> from sklearn import datasets, linear_model >>> from mlxtend.evaluate import bootstrap_point632_score >>> iris = datasets.load_iris() >>> X = iris.data >>> y = iris.target >>> lr = linear_model.LogisticRegression() >>> scores = bootstrap_point632_score(lr, X, y) >>> acc = np.mean(scores) >>> print('Accuracy:', acc) 0.953023146884 >>> lower = np.percentile(scores, 2.5) >>> upper = np.percentile(scores, 97.5) >>> print('95%% Confidence interval: [%.2f, %.2f]' % (lower, upper)) 95% Confidence interval: [0.90, 0.98] For more usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/bootstrap_point632_score/","title":"bootstrap_point632_score: The .632 and .632+ boostrap for classifier evaluation"},{"location":"user_guide/evaluate/bootstrap_point632_score/#bootstrap_point632_score-the-632-and-632-boostrap-for-classifier-evaluation","text":"An implementation of the .632 bootstrap to evaluate supervised learning algorithms. from mlxtend.evaluate import bootstrap_point632_score","title":"bootstrap_point632_score: The .632 and .632+ boostrap for classifier evaluation"},{"location":"user_guide/evaluate/bootstrap_point632_score/#overview","text":"Originally, the bootstrap method aims to determine the statistical properties of an estimator when the underlying distribution was unknown and additional samples are not available. Now, in order to exploit this method for the evaluation of predictive models, such as hypotheses for classification and regression, we may prefer a slightly different approach to bootstrapping using the so-called Out-Of-Bag (OOB) or Leave-One-Out Bootstrap (LOOB) technique. Here, we use out-of-bag samples as test sets for evaluation instead of evaluating the model on the training data. Out-of-bag samples are the unique sets of instances that are not used for model fitting as shown in the figure below [1]. The figure above illustrates how three random bootstrap samples drawn from an exemplary ten-sample dataset ( X_1,X_2, ..., X_{10} ) and their out-of-bag sample for testing may look like. In practice, Bradley Efron and Robert Tibshirani recommend drawing 50 to 200 bootstrap samples as being sufficient for reliable estimates [2].","title":"Overview"},{"location":"user_guide/evaluate/bootstrap_point632_score/#632-bootstrap","text":"In 1983, Bradley Efron described the .632 Estimate , a further improvement to address the pessimistic bias of the bootstrap cross-validation approach described above [3]. The pessimistic bias in the \"classic\" bootstrap method can be attributed to the fact that the bootstrap samples only contain approximately 63.2% of the unique samples from the original dataset. For instance, we can compute the probability that a given sample from a dataset of size n is not drawn as a bootstrap sample as P (\\text{not chosen}) = \\bigg(1 - \\frac{1}{n}\\bigg)^n, which is asymptotically equivalent to \\frac{1}{e} \\approx 0.368 as n \\rightarrow \\infty. Vice versa, we can then compute the probability that a sample is chosen as P (\\text{chosen}) = 1 - \\bigg(1 - \\frac{1}{n}\\bigg)^n \\approx 0.632 for reasonably large datasets, so that we'd select approximately 0.632 \\times n uniques samples as bootstrap training sets and reserve 0.368 \\times n out-of-bag samples for testing in each iteration. Now, to address the bias that is due to this the sampling with replacement, Bradley Efron proposed the .632 Estimate that we mentioned earlier, which is computed via the following equation: \\text{ACC}_{boot} = \\frac{1}{b} \\sum_{i=1}^b \\big(0.632 \\cdot \\text{ACC}_{h, i} + 0.368 \\cdot \\text{ACC}_{train}\\big), where \\text{ACC}_{train} is the accuracy computed on the whole training set, and \\text{ACC}_{h, i} is the accuracy on the out-of-bag sample.","title":".632 Bootstrap"},{"location":"user_guide/evaluate/bootstrap_point632_score/#632-bootstrap_1","text":"Now, while the .632 Boostrap attempts to address the pessimistic bias of the estimate, an optimistic bias may occur with models that tend to overfit so that Bradley Efron and Robert Tibshirani proposed the The .632+ Bootstrap Method (Efron and Tibshirani, 1997). Instead of using a fixed \"weight\" \\omega = 0.632 in ACC_{\\text{boot}} = \\frac{1}{b} \\sum_{i=1}^b \\big(\\omega \\cdot \\text{ACC}_{h, i} + (1-\\omega) \\cdot \\text{ACC}_{train} \\big), we compute the weight \\gamma as \\omega = \\frac{0.632}{1 - 0.368 \\times R}, where R is the relative overfitting rate R = \\frac{(-1) \\times (\\text{ACC}_{h, i} - \\text{ACC}_{train})}{\\gamma - (1 -\\text{ACC}_{h, i})}. (Since we are plugging \\omega into the equation for computing ACC_{boot} that we defined above, \\text{ACC}_{h, i} and \\text{ACC}_{train} still refer to the out-of-bag accuracy in the i th bootstrap round and the whole training set accuracy, respectively.) Further, we need to determine the no-information rate \\gamma in order to compute R . For instance, we can compute \\gamma by fitting a model to a dataset that contains all possible combinations between samples x_{i'} and target class labels y_{i} \u2014 we pretend that the observations and class labels are independent: \\gamma = \\frac{1}{n^2} \\sum_{i=1}^{n} \\sum_{i '=1}^{n} L(y_{i}, f(x_{i '})). Alternatively, we can estimate the no-information rate \\gamma as follows: \\gamma = \\sum_{k=1}^K p_k (1 - q_k), where p_k is the proportion of class k samples observed in the dataset, and q_k is the proportion of class k samples that the classifier predicts in the dataset.","title":".632+ Bootstrap"},{"location":"user_guide/evaluate/bootstrap_point632_score/#references","text":"[1] https://sebastianraschka.com/blog/2016/model-evaluation-selection-part2.html [2] Efron, Bradley, and Robert J. Tibshirani. An introduction to the bootstrap. CRC press, 1994. Management of Data (ACM SIGMOD '97), pages 265-276, 1997. [3] Efron, Bradley. 1983. \u201cEstimating the Error Rate of a Prediction Rule: Improvement on Cross-Validation.\u201d Journal of the American Statistical Association 78 (382): 316. doi:10.2307/2288636. [4] Efron, Bradley, and Robert Tibshirani. 1997. \u201cImprovements on Cross-Validation: The .632+ Bootstrap Method.\u201d Journal of the American Statistical Association 92 (438): 548. doi:10.2307/2965703.","title":"References"},{"location":"user_guide/evaluate/bootstrap_point632_score/#example-1-evaluating-the-predictive-performance-of-a-model-via-the-classic-out-of-bag-bootstrap","text":"The bootstrap_point632_score function mimics the behavior of scikit-learn's `cross_val_score, and a typically usage example is shown below: from sklearn import datasets from sklearn.tree import DecisionTreeClassifier from mlxtend.evaluate import bootstrap_point632_score import numpy as np iris = datasets.load_iris() X = iris.data y = iris.target tree = DecisionTreeClassifier(random_state=123) # Model accuracy scores = bootstrap_point632_score(tree, X, y, method='oob') acc = np.mean(scores) print('Accuracy: %.2f%%' % (100*acc)) # Confidence interval lower = np.percentile(scores, 2.5) upper = np.percentile(scores, 97.5) print('95%% Confidence interval: [%.2f, %.2f]' % (100*lower, 100*upper)) Accuracy: 94.45% 95% Confidence interval: [87.71, 100.00]","title":"Example 1 -- Evaluating the predictive performance of a model via the classic out-of-bag Bootstrap"},{"location":"user_guide/evaluate/bootstrap_point632_score/#example-2-evaluating-the-predictive-performance-of-a-model-via-the-632-bootstrap","text":"from sklearn import datasets from sklearn.tree import DecisionTreeClassifier from mlxtend.evaluate import bootstrap_point632_score import numpy as np iris = datasets.load_iris() X = iris.data y = iris.target tree = DecisionTreeClassifier(random_state=123) # Model accuracy scores = bootstrap_point632_score(tree, X, y) acc = np.mean(scores) print('Accuracy: %.2f%%' % (100*acc)) # Confidence interval lower = np.percentile(scores, 2.5) upper = np.percentile(scores, 97.5) print('95%% Confidence interval: [%.2f, %.2f]' % (100*lower, 100*upper)) Accuracy: 96.42% 95% Confidence interval: [92.41, 100.00]","title":"Example 2 -- Evaluating the predictive performance of a model via the .632 Bootstrap"},{"location":"user_guide/evaluate/bootstrap_point632_score/#example-3-evaluating-the-predictive-performance-of-a-model-via-the-632-bootstrap","text":"from sklearn import datasets from sklearn.tree import DecisionTreeClassifier from mlxtend.evaluate import bootstrap_point632_score import numpy as np iris = datasets.load_iris() X = iris.data y = iris.target tree = DecisionTreeClassifier(random_state=123) # Model accuracy scores = bootstrap_point632_score(tree, X, y, method='.632+') acc = np.mean(scores) print('Accuracy: %.2f%%' % (100*acc)) # Confidence interval lower = np.percentile(scores, 2.5) upper = np.percentile(scores, 97.5) print('95%% Confidence interval: [%.2f, %.2f]' % (100*lower, 100*upper)) Accuracy: 96.29% 95% Confidence interval: [91.86, 98.92]","title":"Example 3 -- Evaluating the predictive performance of a model via the .632+ Bootstrap"},{"location":"user_guide/evaluate/bootstrap_point632_score/#api","text":"bootstrap_point632_score(estimator, X, y, n_splits=200, method='.632', scoring_func=None, predict_proba=False, random_seed=None, clone_estimator=True) Implementation of the .632 [1] and .632+ [2] bootstrap for supervised learning References: - [1] Efron, Bradley. 1983. \"Estimating the Error Rate of a Prediction Rule: Improvement on Cross-Validation.\" Journal of the American Statistical Association 78 (382): 316. doi:10.2307/2288636. - [2] Efron, Bradley, and Robert Tibshirani. 1997. \"Improvements on Cross-Validation: The .632+ Bootstrap Method.\" Journal of the American Statistical Association 92 (438): 548. doi:10.2307/2965703. Parameters estimator : object An estimator for classification or regression that follows the scikit-learn API and implements \"fit\" and \"predict\" methods. X : array-like The data to fit. Can be, for example a list, or an array at least 2d. y : array-like, optional, default: None The target variable to try to predict in the case of supervised learning. n_splits : int (default=200) Number of bootstrap iterations. Must be larger than 1. method : str (default='.632') The bootstrap method, which can be either - 1) '.632' bootstrap (default) - 2) '.632+' bootstrap - 3) 'oob' (regular out-of-bag, no weighting) for comparison studies. scoring_func : callable, Score function (or loss function) with signature scoring_func(y, y_pred, **kwargs) . If none, uses classification accuracy if the estimator is a classifier and mean squared error if the estimator is a regressor. predict_proba : bool Whether to use the predict_proba function for the estimator argument. This is to be used in conjunction with scoring_func which takes in probability values instead of actual predictions. For example, if the scoring_func is :meth: sklearn.metrics.roc_auc_score , then use predict_proba=True . Note that this requires estimator to have predict_proba method implemented. random_seed : int (default=None) If int, random_seed is the seed used by the random number generator. clone_estimator : bool (default=True) Clones the estimator if true, otherwise fits the original. Returns scores : array of float, shape=(len(list(n_splits)),) Array of scores of the estimator for each bootstrap replicate. Examples >>> from sklearn import datasets, linear_model >>> from mlxtend.evaluate import bootstrap_point632_score >>> iris = datasets.load_iris() >>> X = iris.data >>> y = iris.target >>> lr = linear_model.LogisticRegression() >>> scores = bootstrap_point632_score(lr, X, y) >>> acc = np.mean(scores) >>> print('Accuracy:', acc) 0.953023146884 >>> lower = np.percentile(scores, 2.5) >>> upper = np.percentile(scores, 97.5) >>> print('95%% Confidence interval: [%.2f, %.2f]' % (lower, upper)) 95% Confidence interval: [0.90, 0.98] For more usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/bootstrap_point632_score/","title":"API"},{"location":"user_guide/evaluate/cochrans_q/","text":"cochrans_q: Cochran's Q test for comparing multiple classifiers Cochran's Q test for comparing the performance of multiple classifiers. from mlxtend.evaluate import cochrans_q Overview Cochran's Q test can be regarded as a generalized version of McNemar's test that can be applied to evaluate multiple classifiers. In a sense, Cochran's Q test is analogous to ANOVA for binary outcomes. To compare more than two classifiers, we can use Cochran's Q test, which has a test statistic Q that is approximately, (similar to McNemar's test), distributed as chi-squared with L-1 degrees of freedom, where L is the number of models we evaluate (since L=2 for McNemar's test, McNemars test statistic approximates a chi-squared distribution with one degree of freedom). More formally, Cochran's Q test tests the hypothesis that there is no difference between the classification accuracies [1]: p_i: H_0 = p_1 = p_2 = \\cdots = p_L. Let \\{D_1, \\dots , D_L\\} be a set of classifiers who have all been tested on the same dataset. If the L classifiers don't perform differently, then the following Q statistic is distributed approximately as \"chi-squared\" with L-1 degrees of freedom: Q_C = (L-1) \\frac{L \\sum^{L}_{i=1}G_{i}^{2} - T^2}{LT - \\sum^{N_{ts}}_{j=1} (L_j)^2}. Here, G_i is the number of objects out of N_{ts} correctly classified by D_i= 1, \\dots L ; L_j is the number of classifiers out of L that correctly classified object \\mathbf{z}_j \\in \\mathbf{Z}_{ts} , where \\mathbf{Z}_{ts} = \\{\\mathbf{z}_1, ... \\mathbf{z}_{N_{ts}}\\} is the test dataset on which the classifers are tested on; and T is the total number of correct number of votes among the L classifiers [2]: T = \\sum_{i=1}^{L} G_i = \\sum^{N_{ts}}_{j=1} L_j. To perform Cochran's Q test, we typically organize the classificier predictions in a binary N_{ts} \\times L matrix. The ij\\text{th} entry of such matrix is 0 if a classifier D_j has misclassified a data example (vector) \\mathbf{z}_i and 1 otherwise (if the classifier predicted the class label l(\\mathbf{z}_i) correctly) [2]. The following example taken from [2] illustrates how the classification results may be organized. For instance, assume we have the ground truth labels of the test dataset y_true and the following predictions by 3 classifiers ( y_model_1 , y_model_2 , and y_model_3 ): y_true = np.array([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]) y_model_1 = np.array([1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]) y_model_2 = np.array([1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]) y_model_3 = np.array([1, 1, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1]) The table of correct (1) and incorrect (0) classifications may then look as follows: D_1 (model 1) D_2 (model 2) D_3 (model 3) Occurrences 1 1 1 80 1 1 0 2 1 0 1 0 1 0 0 2 0 1 1 9 0 1 0 1 0 0 1 3 0 0 0 3 Accuracy 84/100*100% = 84% 92/100*100% = 92% 92/100*100% = 92% By plugging in the respective value into the previous equation, we obtain the following Q value [2]: Q_c = 2 \\times \\frac{3 \\times (84^2 + 92^2 + 92^2) - 268^2}{3\\times 268-(80 \\times 9 + 11 \\times 4 + 6 \\times 1)} \\approx 7.5294. (Note that the Q value in [2] is listed as 3.7647 due to a typo as discussed with the author, the value 7.5294 is the correct one.) Now, the Q value (approximating \\chi^2 ) corresponds to a p-value of approx. 0.023 assuming a \\chi^2 distribution with L-1 = 2 degrees of freedom. Assuming that we chose a significance level of \\alpha=0.05 , we would reject the null hypothesis that all classifiers perform equally well, since 0.023 < \\alpha . In practice, if we successfully rejected the null hypothesis, we could perform multiple post hoc pair-wise tests -- for example, McNemar tests with a Bonferroni correction -- to determine which pairs have different population proportions. References [1] Fleiss, Joseph L., Bruce Levin, and Myunghee Cho Paik. Statistical methods for rates and proportions. John Wiley & Sons, 2013. [2] Kuncheva, Ludmila I. Combining pattern classifiers: methods and algorithms. John Wiley & Sons, 2004. Example 1 - Cochran's Q test import numpy as np from mlxtend.evaluate import cochrans_q from mlxtend.evaluate import mcnemar_table from mlxtend.evaluate import mcnemar ## Dataset: # ground truth labels of the test dataset: y_true = np.array([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]) # predictions by 3 classifiers (`y_model_1`, `y_model_2`, and `y_model_3`): y_model_1 = np.array([1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]) y_model_2 = np.array([1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]) y_model_3 = np.array([1, 1, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1]) Assuming a significance level \\alpha=0.05 , we can conduct Cochran's Q test as follows, to test the null hypothesis there is no difference between the classification accuracies, p_i: H_0 = p_1 = p_2 = \\cdots = p_L : q, p_value = cochrans_q(y_true, y_model_1, y_model_2, y_model_3) print('Q: %.3f' % q) print('p-value: %.3f' % p_value) Q: 7.529 p-value: 0.023 Since the p-value is smaller than \\alpha , we can reject the null hypothesis and conclude that there is a difference between the classification accuracies. As mentioned in the introduction earlier, we could now perform multiple post hoc pair-wise tests -- for example, McNemar tests with a Bonferroni correction -- to determine which pairs have different population proportions. Lastly, let's illustrate that Cochran's Q test is indeed just a generalized version of McNemar's test: chi2, p_value = cochrans_q(y_true, y_model_1, y_model_2) print('Cochran\\'s Q Chi^2: %.3f' % chi2) print('Cochran\\'s Q p-value: %.3f' % p_value) Cochran's Q Chi^2: 5.333 Cochran's Q p-value: 0.021 chi2, p_value = mcnemar(mcnemar_table(y_true, y_model_1, y_model_2), corrected=False) print('McNemar\\'s Chi^2: %.3f' % chi2) print('McNemar\\'s p-value: %.3f' % p_value) McNemar's Chi^2: 5.333 McNemar's p-value: 0.021 API cochrans_q(y_target, y_model_predictions)* Cochran's Q test to compare 2 or more models. Parameters y_target : array-like, shape=[n_samples] True class labels as 1D NumPy array. *y_model_predictions : array-likes, shape=[n_samples] Variable number of 2 or more arrays that contain the predicted class labels from models as 1D NumPy array. Returns q, p : float or None, float Returns the Q (chi-squared) value and the p-value Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/cochrans_q/","title":"cochrans_q: Cochran's Q test for comparing multiple classifiers"},{"location":"user_guide/evaluate/cochrans_q/#cochrans_q-cochrans-q-test-for-comparing-multiple-classifiers","text":"Cochran's Q test for comparing the performance of multiple classifiers. from mlxtend.evaluate import cochrans_q","title":"cochrans_q: Cochran's Q test for comparing multiple classifiers"},{"location":"user_guide/evaluate/cochrans_q/#overview","text":"Cochran's Q test can be regarded as a generalized version of McNemar's test that can be applied to evaluate multiple classifiers. In a sense, Cochran's Q test is analogous to ANOVA for binary outcomes. To compare more than two classifiers, we can use Cochran's Q test, which has a test statistic Q that is approximately, (similar to McNemar's test), distributed as chi-squared with L-1 degrees of freedom, where L is the number of models we evaluate (since L=2 for McNemar's test, McNemars test statistic approximates a chi-squared distribution with one degree of freedom). More formally, Cochran's Q test tests the hypothesis that there is no difference between the classification accuracies [1]: p_i: H_0 = p_1 = p_2 = \\cdots = p_L. Let \\{D_1, \\dots , D_L\\} be a set of classifiers who have all been tested on the same dataset. If the L classifiers don't perform differently, then the following Q statistic is distributed approximately as \"chi-squared\" with L-1 degrees of freedom: Q_C = (L-1) \\frac{L \\sum^{L}_{i=1}G_{i}^{2} - T^2}{LT - \\sum^{N_{ts}}_{j=1} (L_j)^2}. Here, G_i is the number of objects out of N_{ts} correctly classified by D_i= 1, \\dots L ; L_j is the number of classifiers out of L that correctly classified object \\mathbf{z}_j \\in \\mathbf{Z}_{ts} , where \\mathbf{Z}_{ts} = \\{\\mathbf{z}_1, ... \\mathbf{z}_{N_{ts}}\\} is the test dataset on which the classifers are tested on; and T is the total number of correct number of votes among the L classifiers [2]: T = \\sum_{i=1}^{L} G_i = \\sum^{N_{ts}}_{j=1} L_j. To perform Cochran's Q test, we typically organize the classificier predictions in a binary N_{ts} \\times L matrix. The ij\\text{th} entry of such matrix is 0 if a classifier D_j has misclassified a data example (vector) \\mathbf{z}_i and 1 otherwise (if the classifier predicted the class label l(\\mathbf{z}_i) correctly) [2]. The following example taken from [2] illustrates how the classification results may be organized. For instance, assume we have the ground truth labels of the test dataset y_true and the following predictions by 3 classifiers ( y_model_1 , y_model_2 , and y_model_3 ): y_true = np.array([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]) y_model_1 = np.array([1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]) y_model_2 = np.array([1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]) y_model_3 = np.array([1, 1, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1]) The table of correct (1) and incorrect (0) classifications may then look as follows: D_1 (model 1) D_2 (model 2) D_3 (model 3) Occurrences 1 1 1 80 1 1 0 2 1 0 1 0 1 0 0 2 0 1 1 9 0 1 0 1 0 0 1 3 0 0 0 3 Accuracy 84/100*100% = 84% 92/100*100% = 92% 92/100*100% = 92% By plugging in the respective value into the previous equation, we obtain the following Q value [2]: Q_c = 2 \\times \\frac{3 \\times (84^2 + 92^2 + 92^2) - 268^2}{3\\times 268-(80 \\times 9 + 11 \\times 4 + 6 \\times 1)} \\approx 7.5294. (Note that the Q value in [2] is listed as 3.7647 due to a typo as discussed with the author, the value 7.5294 is the correct one.) Now, the Q value (approximating \\chi^2 ) corresponds to a p-value of approx. 0.023 assuming a \\chi^2 distribution with L-1 = 2 degrees of freedom. Assuming that we chose a significance level of \\alpha=0.05 , we would reject the null hypothesis that all classifiers perform equally well, since 0.023 < \\alpha . In practice, if we successfully rejected the null hypothesis, we could perform multiple post hoc pair-wise tests -- for example, McNemar tests with a Bonferroni correction -- to determine which pairs have different population proportions.","title":"Overview"},{"location":"user_guide/evaluate/cochrans_q/#references","text":"[1] Fleiss, Joseph L., Bruce Levin, and Myunghee Cho Paik. Statistical methods for rates and proportions. John Wiley & Sons, 2013. [2] Kuncheva, Ludmila I. Combining pattern classifiers: methods and algorithms. John Wiley & Sons, 2004.","title":"References"},{"location":"user_guide/evaluate/cochrans_q/#example-1-cochrans-q-test","text":"import numpy as np from mlxtend.evaluate import cochrans_q from mlxtend.evaluate import mcnemar_table from mlxtend.evaluate import mcnemar ## Dataset: # ground truth labels of the test dataset: y_true = np.array([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]) # predictions by 3 classifiers (`y_model_1`, `y_model_2`, and `y_model_3`): y_model_1 = np.array([1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]) y_model_2 = np.array([1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]) y_model_3 = np.array([1, 1, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1]) Assuming a significance level \\alpha=0.05 , we can conduct Cochran's Q test as follows, to test the null hypothesis there is no difference between the classification accuracies, p_i: H_0 = p_1 = p_2 = \\cdots = p_L : q, p_value = cochrans_q(y_true, y_model_1, y_model_2, y_model_3) print('Q: %.3f' % q) print('p-value: %.3f' % p_value) Q: 7.529 p-value: 0.023 Since the p-value is smaller than \\alpha , we can reject the null hypothesis and conclude that there is a difference between the classification accuracies. As mentioned in the introduction earlier, we could now perform multiple post hoc pair-wise tests -- for example, McNemar tests with a Bonferroni correction -- to determine which pairs have different population proportions. Lastly, let's illustrate that Cochran's Q test is indeed just a generalized version of McNemar's test: chi2, p_value = cochrans_q(y_true, y_model_1, y_model_2) print('Cochran\\'s Q Chi^2: %.3f' % chi2) print('Cochran\\'s Q p-value: %.3f' % p_value) Cochran's Q Chi^2: 5.333 Cochran's Q p-value: 0.021 chi2, p_value = mcnemar(mcnemar_table(y_true, y_model_1, y_model_2), corrected=False) print('McNemar\\'s Chi^2: %.3f' % chi2) print('McNemar\\'s p-value: %.3f' % p_value) McNemar's Chi^2: 5.333 McNemar's p-value: 0.021","title":"Example 1 - Cochran's Q test"},{"location":"user_guide/evaluate/cochrans_q/#api","text":"cochrans_q(y_target, y_model_predictions)* Cochran's Q test to compare 2 or more models. Parameters y_target : array-like, shape=[n_samples] True class labels as 1D NumPy array. *y_model_predictions : array-likes, shape=[n_samples] Variable number of 2 or more arrays that contain the predicted class labels from models as 1D NumPy array. Returns q, p : float or None, float Returns the Q (chi-squared) value and the p-value Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/cochrans_q/","title":"API"},{"location":"user_guide/evaluate/combined_ftest_5x2cv/","text":"combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons 5x2cv combined F test procedure to compare the performance of two models from mlxtend.evaluate import combined_ftest_5x2cv Overview The 5x2cv combined F test is a procedure for comparing the performance of two models (classifiers or regressors) that was proposed by Alpaydin [1] as a more robust alternative to Dietterich's 5x2cv paired t-test procedure [2]. paired_ttest_5x2cv.md . Dietterich's 5x2cv method was in turn designed to address shortcomings in other methods such as the resampled paired t test (see paired_ttest_resampled ) and the k-fold cross-validated paired t test (see paired_ttest_kfold_cv ). To explain how this method works, let's consider to estimator (e.g., classifiers) A and B. Further, we have a labeled dataset D . In the common hold-out method, we typically split the dataset into 2 parts: a training and a test set. In the 5x2cv paired t test, we repeat the splitting (50% training and 50% test data) 5 times. In each of the 5 iterations, we fit A and B to the training split and evaluate their performance ( p_A and p_B ) on the test split. Then, we rotate the training and test sets (the training set becomes the test set and vice versa) compute the performance again, which results in 2 performance difference measures: p^{(1)} = p^{(1)}_A - p^{(1)}_B and p^{(2)} = p^{(2)}_A - p^{(2)}_B. Then, we estimate mean and variance of the differences: \\overline{p} = \\frac{p^{(1)} + p^{(2)}}{2} and s^2 = (p^{(1)} - \\overline{p})^2 + (p^{(2)} - \\overline{p})^2. The F-statistic proposed by Alpaydin (see paper for justifications) is then computed as \\mathcal{f} = \\frac{\\sum_{i=1}^{5} \\sum_{j=1}^2 (p_i^{j})^2}{2 \\sum_{i=1}^5 s_i^2}, which is approximately F distributed with 10 and 5 degress of freedom. Using the f statistic, the p value can be computed and compared with a previously chosen significance level, e.g., \\alpha=0.05 . If the p value is smaller than \\alpha , we reject the null hypothesis and accept that there is a significant difference in the two models. References [1] Alpaydin, E. (1999). Combined 5\u00d72 cv F test for comparing supervised classification learning algorithms. Neural computation, 11(8), 1885-1892. [2] Dietterich TG (1998) Approximate Statistical Tests for Comparing Supervised Classification Learning Algorithms. Neural Comput 10:1895\u20131923. Example 1 - 5x2cv combined F test Assume we want to compare two classification algorithms, logistic regression and a decision tree algorithm: from sklearn.linear_model import LogisticRegression from sklearn.tree import DecisionTreeClassifier from mlxtend.data import iris_data from sklearn.model_selection import train_test_split X, y = iris_data() clf1 = LogisticRegression(random_state=1, solver='liblinear', multi_class='ovr') clf2 = DecisionTreeClassifier(random_state=1) X_train, X_test, y_train, y_test = \\ train_test_split(X, y, test_size=0.25, random_state=123) score1 = clf1.fit(X_train, y_train).score(X_test, y_test) score2 = clf2.fit(X_train, y_train).score(X_test, y_test) print('Logistic regression accuracy: %.2f%%' % (score1*100)) print('Decision tree accuracy: %.2f%%' % (score2*100)) Logistic regression accuracy: 97.37% Decision tree accuracy: 94.74% Note that these accuracy values are not used in the paired f test procedure as new test/train splits are generated during the resampling procedure, the values above are just serving the purpose of intuition. Now, let's assume a significance threshold of \\alpha=0.05 for rejecting the null hypothesis that both algorithms perform equally well on the dataset and conduct the 5x2cv f test: from mlxtend.evaluate import combined_ftest_5x2cv f, p = combined_ftest_5x2cv(estimator1=clf1, estimator2=clf2, X=X, y=y, random_seed=1) print('F statistic: %.3f' % f) print('p value: %.3f' % p) F statistic: 1.053 p value: 0.509 Since p > \\alpha , we cannot reject the null hypothesis and may conclude that the performance of the two algorithms is not significantly different. While it is generally not recommended to apply statistical tests multiple times without correction for multiple hypothesis testing, let us take a look at an example where the decision tree algorithm is limited to producing a very simple decision boundary that would result in a relatively bad performance: clf2 = DecisionTreeClassifier(random_state=1, max_depth=1) score2 = clf2.fit(X_train, y_train).score(X_test, y_test) print('Decision tree accuracy: %.2f%%' % (score2*100)) f, p = combined_ftest_5x2cv(estimator1=clf1, estimator2=clf2, X=X, y=y, random_seed=1) print('F statistic: %.3f' % f) print('p value: %.3f' % p) Decision tree accuracy: 63.16% F statistic: 34.934 p value: 0.001 Assuming that we conducted this test also with a significance level of \\alpha=0.05 , we can reject the null-hypothesis that both models perform equally well on this dataset, since the p-value ( p < 0.001 ) is smaller than \\alpha . API combined_ftest_5x2cv(estimator1, estimator2, X, y, scoring=None, random_seed=None) Implements the 5x2cv combined F test proposed by Alpaydin 1999, to compare the performance of two models. Parameters estimator1 : scikit-learn classifier or regressor estimator2 : scikit-learn classifier or regressor X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values. scoring : str, callable, or None (default: None) If None (default), uses 'accuracy' for sklearn classifiers and 'r2' for sklearn regressors. If str, uses a sklearn scoring metric string identifier, for example {accuracy, f1, precision, recall, roc_auc} for classifiers, {'mean_absolute_error', 'mean_squared_error'/'neg_mean_squared_error', 'median_absolute_error', 'r2'} for regressors. If a callable object or function is provided, it has to be conform with sklearn's signature scorer(estimator, X, y) ; see https://scikit-learn.org/stable/modules/generated/sklearn.metrics.make_scorer.html for more information. random_seed : int or None (default: None) Random seed for creating the test/train splits. Returns f : float The F-statistic pvalue : float Two-tailed p-value. If the chosen significance level is larger than the p-value, we reject the null hypothesis and accept that there are significant differences in the two compared models. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/combined_ftest_5x2cv/","title":"combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons"},{"location":"user_guide/evaluate/combined_ftest_5x2cv/#combined_ftest_5x2cv-5x2cv-combined-f-test-for-classifier-comparisons","text":"5x2cv combined F test procedure to compare the performance of two models from mlxtend.evaluate import combined_ftest_5x2cv","title":"combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons"},{"location":"user_guide/evaluate/combined_ftest_5x2cv/#overview","text":"The 5x2cv combined F test is a procedure for comparing the performance of two models (classifiers or regressors) that was proposed by Alpaydin [1] as a more robust alternative to Dietterich's 5x2cv paired t-test procedure [2]. paired_ttest_5x2cv.md . Dietterich's 5x2cv method was in turn designed to address shortcomings in other methods such as the resampled paired t test (see paired_ttest_resampled ) and the k-fold cross-validated paired t test (see paired_ttest_kfold_cv ). To explain how this method works, let's consider to estimator (e.g., classifiers) A and B. Further, we have a labeled dataset D . In the common hold-out method, we typically split the dataset into 2 parts: a training and a test set. In the 5x2cv paired t test, we repeat the splitting (50% training and 50% test data) 5 times. In each of the 5 iterations, we fit A and B to the training split and evaluate their performance ( p_A and p_B ) on the test split. Then, we rotate the training and test sets (the training set becomes the test set and vice versa) compute the performance again, which results in 2 performance difference measures: p^{(1)} = p^{(1)}_A - p^{(1)}_B and p^{(2)} = p^{(2)}_A - p^{(2)}_B. Then, we estimate mean and variance of the differences: \\overline{p} = \\frac{p^{(1)} + p^{(2)}}{2} and s^2 = (p^{(1)} - \\overline{p})^2 + (p^{(2)} - \\overline{p})^2. The F-statistic proposed by Alpaydin (see paper for justifications) is then computed as \\mathcal{f} = \\frac{\\sum_{i=1}^{5} \\sum_{j=1}^2 (p_i^{j})^2}{2 \\sum_{i=1}^5 s_i^2}, which is approximately F distributed with 10 and 5 degress of freedom. Using the f statistic, the p value can be computed and compared with a previously chosen significance level, e.g., \\alpha=0.05 . If the p value is smaller than \\alpha , we reject the null hypothesis and accept that there is a significant difference in the two models.","title":"Overview"},{"location":"user_guide/evaluate/combined_ftest_5x2cv/#references","text":"[1] Alpaydin, E. (1999). Combined 5\u00d72 cv F test for comparing supervised classification learning algorithms. Neural computation, 11(8), 1885-1892. [2] Dietterich TG (1998) Approximate Statistical Tests for Comparing Supervised Classification Learning Algorithms. Neural Comput 10:1895\u20131923.","title":"References"},{"location":"user_guide/evaluate/combined_ftest_5x2cv/#example-1-5x2cv-combined-f-test","text":"Assume we want to compare two classification algorithms, logistic regression and a decision tree algorithm: from sklearn.linear_model import LogisticRegression from sklearn.tree import DecisionTreeClassifier from mlxtend.data import iris_data from sklearn.model_selection import train_test_split X, y = iris_data() clf1 = LogisticRegression(random_state=1, solver='liblinear', multi_class='ovr') clf2 = DecisionTreeClassifier(random_state=1) X_train, X_test, y_train, y_test = \\ train_test_split(X, y, test_size=0.25, random_state=123) score1 = clf1.fit(X_train, y_train).score(X_test, y_test) score2 = clf2.fit(X_train, y_train).score(X_test, y_test) print('Logistic regression accuracy: %.2f%%' % (score1*100)) print('Decision tree accuracy: %.2f%%' % (score2*100)) Logistic regression accuracy: 97.37% Decision tree accuracy: 94.74% Note that these accuracy values are not used in the paired f test procedure as new test/train splits are generated during the resampling procedure, the values above are just serving the purpose of intuition. Now, let's assume a significance threshold of \\alpha=0.05 for rejecting the null hypothesis that both algorithms perform equally well on the dataset and conduct the 5x2cv f test: from mlxtend.evaluate import combined_ftest_5x2cv f, p = combined_ftest_5x2cv(estimator1=clf1, estimator2=clf2, X=X, y=y, random_seed=1) print('F statistic: %.3f' % f) print('p value: %.3f' % p) F statistic: 1.053 p value: 0.509 Since p > \\alpha , we cannot reject the null hypothesis and may conclude that the performance of the two algorithms is not significantly different. While it is generally not recommended to apply statistical tests multiple times without correction for multiple hypothesis testing, let us take a look at an example where the decision tree algorithm is limited to producing a very simple decision boundary that would result in a relatively bad performance: clf2 = DecisionTreeClassifier(random_state=1, max_depth=1) score2 = clf2.fit(X_train, y_train).score(X_test, y_test) print('Decision tree accuracy: %.2f%%' % (score2*100)) f, p = combined_ftest_5x2cv(estimator1=clf1, estimator2=clf2, X=X, y=y, random_seed=1) print('F statistic: %.3f' % f) print('p value: %.3f' % p) Decision tree accuracy: 63.16% F statistic: 34.934 p value: 0.001 Assuming that we conducted this test also with a significance level of \\alpha=0.05 , we can reject the null-hypothesis that both models perform equally well on this dataset, since the p-value ( p < 0.001 ) is smaller than \\alpha .","title":"Example 1 - 5x2cv combined F test"},{"location":"user_guide/evaluate/combined_ftest_5x2cv/#api","text":"combined_ftest_5x2cv(estimator1, estimator2, X, y, scoring=None, random_seed=None) Implements the 5x2cv combined F test proposed by Alpaydin 1999, to compare the performance of two models. Parameters estimator1 : scikit-learn classifier or regressor estimator2 : scikit-learn classifier or regressor X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values. scoring : str, callable, or None (default: None) If None (default), uses 'accuracy' for sklearn classifiers and 'r2' for sklearn regressors. If str, uses a sklearn scoring metric string identifier, for example {accuracy, f1, precision, recall, roc_auc} for classifiers, {'mean_absolute_error', 'mean_squared_error'/'neg_mean_squared_error', 'median_absolute_error', 'r2'} for regressors. If a callable object or function is provided, it has to be conform with sklearn's signature scorer(estimator, X, y) ; see https://scikit-learn.org/stable/modules/generated/sklearn.metrics.make_scorer.html for more information. random_seed : int or None (default: None) Random seed for creating the test/train splits. Returns f : float The F-statistic pvalue : float Two-tailed p-value. If the chosen significance level is larger than the p-value, we reject the null hypothesis and accept that there are significant differences in the two compared models. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/combined_ftest_5x2cv/","title":"API"},{"location":"user_guide/evaluate/confusion_matrix/","text":"confusion_matrix: creating a confusion matrix for model evaluation Functions for generating confusion matrices. from mlxtend.evaluate import confusion_matrix from mlxtend.plotting import plot_confusion_matrix Overview Confusion Matrix The confusion matrix (or error matrix ) is one way to summarize the performance of a classifier for binary classification tasks. This square matrix consists of columns and rows that list the number of instances as absolute or relative \"actual class\" vs. \"predicted class\" ratios. Let P be the label of class 1 and N be the label of a second class or the label of all classes that are not class 1 in a multi-class setting. References - Example 1 - Binary classification from mlxtend.evaluate import confusion_matrix y_target = [0, 0, 1, 0, 0, 1, 1, 1] y_predicted = [1, 0, 1, 0, 0, 0, 0, 1] cm = confusion_matrix(y_target=y_target, y_predicted=y_predicted) cm array([[3, 1], [2, 2]]) To visualize the confusion matrix using matplotlib, see the utility function mlxtend.plotting.plot_confusion_matrix : import matplotlib.pyplot as plt from mlxtend.plotting import plot_confusion_matrix fig, ax = plot_confusion_matrix(conf_mat=cm) plt.show() Example 2 - Multi-class classification from mlxtend.evaluate import confusion_matrix y_target = [1, 1, 1, 0, 0, 2, 0, 3] y_predicted = [1, 0, 1, 0, 0, 2, 1, 3] cm = confusion_matrix(y_target=y_target, y_predicted=y_predicted, binary=False) cm array([[2, 1, 0, 0], [1, 2, 0, 0], [0, 0, 1, 0], [0, 0, 0, 1]]) To visualize the confusion matrix using matplotlib, see the utility function mlxtend.plotting.plot_confusion_matrix : import matplotlib.pyplot as plt from mlxtend.evaluate import confusion_matrix fig, ax = plot_confusion_matrix(conf_mat=cm) plt.show() Example 3 - Multi-class to binary By setting binary=True , all class labels that are not the positive class label are being summarized to class 0. The positive class label becomes class 1. import matplotlib.pyplot as plt from mlxtend.evaluate import confusion_matrix y_target = [1, 1, 1, 0, 0, 2, 0, 3] y_predicted = [1, 0, 1, 0, 0, 2, 1, 3] cm = confusion_matrix(y_target=y_target, y_predicted=y_predicted, binary=True, positive_label=1) cm array([[4, 1], [1, 2]]) To visualize the confusion matrix using matplotlib, see the utility function mlxtend.plotting.plot_confusion_matrix : from mlxtend.plotting import plot_confusion_matrix fig, ax = plot_confusion_matrix(conf_mat=cm) plt.show() API confusion_matrix(y_target, y_predicted, binary=False, positive_label=1) Compute a confusion matrix/contingency table. Parameters y_target : array-like, shape=[n_samples] True class labels. y_predicted : array-like, shape=[n_samples] Predicted class labels. binary : bool (default: False) Maps a multi-class problem onto a binary confusion matrix, where the positive class is 1 and all other classes are 0. positive_label : int (default: 1) Class label of the positive class. Returns mat : array-like, shape=[n_classes, n_classes] Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/confusion_matrix/","title":"confusion_matrix: creating a confusion matrix for model evaluation"},{"location":"user_guide/evaluate/confusion_matrix/#confusion_matrix-creating-a-confusion-matrix-for-model-evaluation","text":"Functions for generating confusion matrices. from mlxtend.evaluate import confusion_matrix from mlxtend.plotting import plot_confusion_matrix","title":"confusion_matrix: creating a confusion matrix for model evaluation"},{"location":"user_guide/evaluate/confusion_matrix/#overview","text":"","title":"Overview"},{"location":"user_guide/evaluate/confusion_matrix/#confusion-matrix","text":"The confusion matrix (or error matrix ) is one way to summarize the performance of a classifier for binary classification tasks. This square matrix consists of columns and rows that list the number of instances as absolute or relative \"actual class\" vs. \"predicted class\" ratios. Let P be the label of class 1 and N be the label of a second class or the label of all classes that are not class 1 in a multi-class setting.","title":"Confusion Matrix"},{"location":"user_guide/evaluate/confusion_matrix/#references","text":"-","title":"References"},{"location":"user_guide/evaluate/confusion_matrix/#example-1-binary-classification","text":"from mlxtend.evaluate import confusion_matrix y_target = [0, 0, 1, 0, 0, 1, 1, 1] y_predicted = [1, 0, 1, 0, 0, 0, 0, 1] cm = confusion_matrix(y_target=y_target, y_predicted=y_predicted) cm array([[3, 1], [2, 2]]) To visualize the confusion matrix using matplotlib, see the utility function mlxtend.plotting.plot_confusion_matrix : import matplotlib.pyplot as plt from mlxtend.plotting import plot_confusion_matrix fig, ax = plot_confusion_matrix(conf_mat=cm) plt.show()","title":"Example 1 - Binary classification"},{"location":"user_guide/evaluate/confusion_matrix/#example-2-multi-class-classification","text":"from mlxtend.evaluate import confusion_matrix y_target = [1, 1, 1, 0, 0, 2, 0, 3] y_predicted = [1, 0, 1, 0, 0, 2, 1, 3] cm = confusion_matrix(y_target=y_target, y_predicted=y_predicted, binary=False) cm array([[2, 1, 0, 0], [1, 2, 0, 0], [0, 0, 1, 0], [0, 0, 0, 1]]) To visualize the confusion matrix using matplotlib, see the utility function mlxtend.plotting.plot_confusion_matrix : import matplotlib.pyplot as plt from mlxtend.evaluate import confusion_matrix fig, ax = plot_confusion_matrix(conf_mat=cm) plt.show()","title":"Example 2 - Multi-class classification"},{"location":"user_guide/evaluate/confusion_matrix/#example-3-multi-class-to-binary","text":"By setting binary=True , all class labels that are not the positive class label are being summarized to class 0. The positive class label becomes class 1. import matplotlib.pyplot as plt from mlxtend.evaluate import confusion_matrix y_target = [1, 1, 1, 0, 0, 2, 0, 3] y_predicted = [1, 0, 1, 0, 0, 2, 1, 3] cm = confusion_matrix(y_target=y_target, y_predicted=y_predicted, binary=True, positive_label=1) cm array([[4, 1], [1, 2]]) To visualize the confusion matrix using matplotlib, see the utility function mlxtend.plotting.plot_confusion_matrix : from mlxtend.plotting import plot_confusion_matrix fig, ax = plot_confusion_matrix(conf_mat=cm) plt.show()","title":"Example 3 - Multi-class to binary"},{"location":"user_guide/evaluate/confusion_matrix/#api","text":"confusion_matrix(y_target, y_predicted, binary=False, positive_label=1) Compute a confusion matrix/contingency table. Parameters y_target : array-like, shape=[n_samples] True class labels. y_predicted : array-like, shape=[n_samples] Predicted class labels. binary : bool (default: False) Maps a multi-class problem onto a binary confusion matrix, where the positive class is 1 and all other classes are 0. positive_label : int (default: 1) Class label of the positive class. Returns mat : array-like, shape=[n_classes, n_classes] Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/confusion_matrix/","title":"API"},{"location":"user_guide/evaluate/create_counterfactual/","text":"create_counterfactual: Interpreting models via counterfactuals An implementation of the counterfactual method by Wachter et al. 2017 for model interpretability. from mlxtend.evaluate import create_counterfactual Overview Counterfactuals are instances that explain scenarios related to implication: \"if not x, then not y\" in a hypothetical context. For example, \"if I hadn't studied hard, my grade would be worse.\" In the context of machine learning, we can think of counterfactual instances from the training set for which we artificially change its features to change the model prediction. Changing features of a training example can be useful to interpret the behavior of the model. Note that this implementation for creating counterfactuals is model agnostic and works with any scikit-learn estimators that support the predict (and ideally predict_proba ) method. In particular, the create_counterfactual implements the method described by Wachter et al. 2017 [1]. A good, short description of this method is also available in C. Molnar's Interpretable Machine Learning Book [2]. In short, Wachter et al.'s method minimizes the loss L\\left(x, x^{\\prime}, y^{\\prime}, \\lambda\\right)=\\lambda \\cdot\\left(\\hat{f}\\left(x^{\\prime}\\right)-y^{\\prime}\\right)^{2}+d\\left(x, x^{\\prime}\\right). The left term, \\lambda \\cdot\\left(\\hat{f}\\left(x^{\\prime}\\right)-y^{\\prime}\\right)^{2} , minimizes the squared difference between the model prediction for the counterfactual x' , i.e., \\hat{f}\\left(x^{\\prime}\\right) , and the desired prediction (specified by the user), y^{\\prime} . Note that \\lambda is a hyperparameter for weighting the importance of this left term over the second term, d\\left(x, x^{\\prime}\\right) . The second term, d\\left(x, x^{\\prime}\\right) , calculates the distance between a given instance x and a generated counterfactual x' . In short, the second term will keep the generated counterfactual similar to the instance. In contrast, the first term maximizes the difference between the model prediction for the counterfactual and the desired prediction (for example, a different class label). The distance function is implemented as the absolute difference in each feature dimension scaled by the median absolute deviation (MAD): d\\left(x, x^{\\prime}\\right)=\\sum_{j=1}^{p} \\frac{\\left|x_{j}-x_{j}^{\\prime}\\right|}{M A D_{j}}. The MAD measures the spread of a given feature, using the median as its center: MAD_{j}=\\operatorname{median}_{i \\in\\{1, \\ldots, n\\}}\\left(\\left|x_{i, j}-\\operatorname{median}_{l \\in\\{1, \\ldots, n\\}}\\left(x_{l, j}\\right)\\right|\\right). The general procedure for using the create_counterfactual function is as follows. Select an instance that you want to explain and specify the desired prediction y' for this instance (this is usually different from its original prediction). Choose a value for the hyperparameter \\lambda Optimize the loss L using the create_counterfactual function Optionally, as the authors recommend, you can repeat steps 2 and 3 by increasing \\lambda until a user-defined threshold \\epsilon is reached, i.e., while \\left|\\hat{f}\\left(x^{\\prime}\\right)-y^{\\prime}\\right|>\\epsilon : increase \\lambda References [1] Wachter, S., Mittelstadt, B., & Russell, C. (2017). Counterfactual explanations without opening the black box: Automated decisions and the GDPR. Harv. JL & Tech., 31, 841., https://arxiv.org/abs/1711.00399 [2] Christoph Molnar (2018). Interpretable Machine Learning, Chapter 6.1 Example 1 -- Simple Iris Example For simplicity, this example illustrates how to use the create_counterfactual function to explain a data instance from the iris dataset. Suppose we trained a logistic regression model on the iris dataset and pick the 16th training point for which we want to explain the prediction via counterfactuals. from mlxtend.data import iris_data from sklearn.linear_model import LogisticRegression X, y = iris_data() clf = LogisticRegression() clf.fit(X, y) x_ref = X[15] print('True label:', y[15]) print('Predicted label:', clf.predict(x_ref.reshape(1, -1))[0]) print('Predicted probas:', clf.predict_proba(x_ref.reshape(1, -1))) print('Predicted probability for label 0:', clf.predict_proba(x_ref.reshape(1, -1))[0][0]) True label: 0 Predicted label: 0 Predicted probas: [[9.86677291e-01 1.33226960e-02 1.28980184e-08]] Predicted probability for label 0: 0.9866772910539873 We can see above, that there is a predicted score of 98.6% probability for a class 0 membership. Now, we are going to push the prediction towards class 2 by setting y_desired=2 . Moreover, we set the probability for class 2 to 100% via y_desired_proba=1. from mlxtend.evaluate import create_counterfactual res = create_counterfactual(x_reference=x_ref, y_desired=2, model=clf, X_dataset=X, y_desired_proba=1., lammbda=1, # hyperparameter random_seed=123) print('Features of the 16th training example:', x_ref) print('Features of the countefactual:', res) print('Predictions for counterfactual:\\n') print('Predicted label:', clf.predict(res.reshape(1, -1))[0]) print('Predicted probas:', clf.predict_proba(res.reshape(1, -1))) Features of the 16th training example: [5.7 4.4 1.5 0.4] Features of the countefactual: [5.72271344 3.99169005 6.45305374 0.40000002] Predictions for counterfactual: Predicted label: 2 Predicted probas: [[1.41639932e-04 3.13292297e-01 6.86566063e-01]] As we can see above, the counterfactual is relatively similar to the original training example, i.e, only the 3rd feature has changed substantially (from 1.5 to 6.45). The predicted label has changed from class 0 t class 2. Interpretation-wise, this means increasing the petal length of a Iris-setosa flower may make it more similar to a Iris-virginica flower. Example 2 -- Simple Iris Example with Decision Regions and Threshold Stopping Criterion This example is similar to Example 1; however, it is based on a 2D iris dataset containing only petal length and petal width features so that the results can be plotted via a decision region plot. from mlxtend.plotting import plot_decision_regions import matplotlib.pyplot as plt X, y = iris_data() X = X[:, 2:] clf = LogisticRegression() clf.fit(X, y) LogisticRegression() # Plotting decision regions ax = plot_decision_regions(X, y, clf=clf, legend=2) scatter_highlight_defaults = {'c': 'red', 'edgecolor': 'yellow', 'alpha': 1.0, 'linewidths': 2, 'marker': 'o', 's': 80} ax.scatter(*X[15], **scatter_highlight_defaults) plt.show() The big, highlighted point in the plot above shows the 16th training datapoint. The following code will create a counterfactual with the same settings as in Example 1: counterfact = create_counterfactual(x_reference=X[15], y_desired=2, model=clf, X_dataset=X, y_desired_proba=1.0, lammbda=1, random_seed=123) ax = plot_decision_regions(X, y, clf=clf, legend=2) ax.scatter(*counterfact, **scatter_highlight_defaults) plt.show() As we can see above, the counterfactual primarily moved along the x-axis (petal length) so that the prediction between the rerence point and the counterfactual changes from class 0 to class 2. The following plots are based on repeating this procedure with different lambda values: for i in [0.4, 0.5, 1.0, 5.0, 100]: counterfact = create_counterfactual(x_reference=X[15], y_desired=2, model=clf, X_dataset=X, y_desired_proba=1.0, lammbda=i, random_seed=123) ax = plot_decision_regions(X, y, clf=clf, legend=2) ax.scatter(*counterfact, **scatter_highlight_defaults) plt.show() As we can see, the stronger the \\lambda value, the more the first term in the loss L\\left(x, x^{\\prime}, y^{\\prime}, \\lambda\\right)=\\lambda \\cdot\\left(\\hat{f}\\left(x^{\\prime}\\right)-y^{\\prime}\\right)^{2}+d\\left(x, x^{\\prime}\\right). dominates. Applying Wachter et al.'s threshold concept, Optionally, as the authors recommend, you can repeat steps 2 and 3 by increasing \\lambda until a user-defined threshold \\epsilon is reached, i.e., while \\left|\\hat{f}\\left(x^{\\prime}\\right)-y^{\\prime}\\right|>\\epsilon : increase \\lambda we can define a user-defined threshold and implement it as follows: import numpy as np desired_class_2_proba = 1.0 for i in np.arange(0, 10000, 0.1): counterfact = create_counterfactual(x_reference=X[15], y_desired=2, model=clf, X_dataset=X, y_desired_proba=desired_class_2_proba, lammbda=i, random_seed=123) predicted_class_2_proba = clf.predict_proba(counterfact.reshape(1, -1))[0][2] if not i: print('Initial lambda:', i) print('Initial diff:', np.abs(predicted_class_2_proba - desired_class_2_proba)) if not np.abs(predicted_class_2_proba - desired_class_2_proba) > 0.3: break ax = plot_decision_regions(X, y, clf=clf, legend=2) ax.scatter(*counterfact, **scatter_highlight_defaults) print('Final lambda:', i) print('Final diff:', np.abs(predicted_class_2_proba - desired_class_2_proba)) plt.show() Initial lambda: 0.0 Initial diff: 0.9999998976132334 Final lambda: 1.1 Final diff: 0.2962621523225484 API create_counterfactual(x_reference, y_desired, model, X_dataset, y_desired_proba=None, lammbda=0.1, random_seed=None) Implementation of the counterfactual method by Wachter et al. References: Wachter, S., Mittelstadt, B., & Russell, C. (2017). Counterfactual explanations without opening the black box: Automated decisions and the GDPR. Harv. JL & Tech., 31, 841., https://arxiv.org/abs/1711.00399 Parameters x_reference : array-like, shape=[m_features] The data instance (training example) to be explained. y_desired : int The desired class label for x_reference . model : estimator A (scikit-learn) estimator implementing .predict() and/or predict_proba() . - If model supports predict_proba() , then this is used by default for the first loss term, (lambda * model.predict[_proba](x_counterfact) - y_desired[_proba])^2 - Otherwise, method will fall back to predict . X_dataset : array-like, shape=[n_examples, m_features] A (training) dataset for picking the initial counterfactual as initial value for starting the optimization procedure. y_desired_proba : float (default: None) A float within the range [0, 1] designating the desired class probability for y_desired . - If y_desired_proba=None (default), the first loss term is (lambda * model(x_counterfact) - y_desired)^2 where y_desired is a class label - If y_desired_proba is not None, the first loss term is (lambda * model(x_counterfact) - y_desired_proba)^2 lammbda : Weighting parameter for the first loss term, (lambda * model(x_counterfact) - y_desired[_proba])^2 random_seed : int (default=None) If int, random_seed is the seed used by the random number generator for selecting the inital counterfactual from X_dataset . ython","title":"create_counterfactual: Interpreting models via counterfactuals"},{"location":"user_guide/evaluate/create_counterfactual/#create_counterfactual-interpreting-models-via-counterfactuals","text":"An implementation of the counterfactual method by Wachter et al. 2017 for model interpretability. from mlxtend.evaluate import create_counterfactual","title":"create_counterfactual: Interpreting models via counterfactuals"},{"location":"user_guide/evaluate/create_counterfactual/#overview","text":"Counterfactuals are instances that explain scenarios related to implication: \"if not x, then not y\" in a hypothetical context. For example, \"if I hadn't studied hard, my grade would be worse.\" In the context of machine learning, we can think of counterfactual instances from the training set for which we artificially change its features to change the model prediction. Changing features of a training example can be useful to interpret the behavior of the model. Note that this implementation for creating counterfactuals is model agnostic and works with any scikit-learn estimators that support the predict (and ideally predict_proba ) method. In particular, the create_counterfactual implements the method described by Wachter et al. 2017 [1]. A good, short description of this method is also available in C. Molnar's Interpretable Machine Learning Book [2]. In short, Wachter et al.'s method minimizes the loss L\\left(x, x^{\\prime}, y^{\\prime}, \\lambda\\right)=\\lambda \\cdot\\left(\\hat{f}\\left(x^{\\prime}\\right)-y^{\\prime}\\right)^{2}+d\\left(x, x^{\\prime}\\right). The left term, \\lambda \\cdot\\left(\\hat{f}\\left(x^{\\prime}\\right)-y^{\\prime}\\right)^{2} , minimizes the squared difference between the model prediction for the counterfactual x' , i.e., \\hat{f}\\left(x^{\\prime}\\right) , and the desired prediction (specified by the user), y^{\\prime} . Note that \\lambda is a hyperparameter for weighting the importance of this left term over the second term, d\\left(x, x^{\\prime}\\right) . The second term, d\\left(x, x^{\\prime}\\right) , calculates the distance between a given instance x and a generated counterfactual x' . In short, the second term will keep the generated counterfactual similar to the instance. In contrast, the first term maximizes the difference between the model prediction for the counterfactual and the desired prediction (for example, a different class label). The distance function is implemented as the absolute difference in each feature dimension scaled by the median absolute deviation (MAD): d\\left(x, x^{\\prime}\\right)=\\sum_{j=1}^{p} \\frac{\\left|x_{j}-x_{j}^{\\prime}\\right|}{M A D_{j}}. The MAD measures the spread of a given feature, using the median as its center: MAD_{j}=\\operatorname{median}_{i \\in\\{1, \\ldots, n\\}}\\left(\\left|x_{i, j}-\\operatorname{median}_{l \\in\\{1, \\ldots, n\\}}\\left(x_{l, j}\\right)\\right|\\right). The general procedure for using the create_counterfactual function is as follows. Select an instance that you want to explain and specify the desired prediction y' for this instance (this is usually different from its original prediction). Choose a value for the hyperparameter \\lambda Optimize the loss L using the create_counterfactual function Optionally, as the authors recommend, you can repeat steps 2 and 3 by increasing \\lambda until a user-defined threshold \\epsilon is reached, i.e., while \\left|\\hat{f}\\left(x^{\\prime}\\right)-y^{\\prime}\\right|>\\epsilon : increase \\lambda","title":"Overview"},{"location":"user_guide/evaluate/create_counterfactual/#references","text":"[1] Wachter, S., Mittelstadt, B., & Russell, C. (2017). Counterfactual explanations without opening the black box: Automated decisions and the GDPR. Harv. JL & Tech., 31, 841., https://arxiv.org/abs/1711.00399 [2] Christoph Molnar (2018). Interpretable Machine Learning, Chapter 6.1","title":"References"},{"location":"user_guide/evaluate/create_counterfactual/#example-1-simple-iris-example","text":"For simplicity, this example illustrates how to use the create_counterfactual function to explain a data instance from the iris dataset. Suppose we trained a logistic regression model on the iris dataset and pick the 16th training point for which we want to explain the prediction via counterfactuals. from mlxtend.data import iris_data from sklearn.linear_model import LogisticRegression X, y = iris_data() clf = LogisticRegression() clf.fit(X, y) x_ref = X[15] print('True label:', y[15]) print('Predicted label:', clf.predict(x_ref.reshape(1, -1))[0]) print('Predicted probas:', clf.predict_proba(x_ref.reshape(1, -1))) print('Predicted probability for label 0:', clf.predict_proba(x_ref.reshape(1, -1))[0][0]) True label: 0 Predicted label: 0 Predicted probas: [[9.86677291e-01 1.33226960e-02 1.28980184e-08]] Predicted probability for label 0: 0.9866772910539873 We can see above, that there is a predicted score of 98.6% probability for a class 0 membership. Now, we are going to push the prediction towards class 2 by setting y_desired=2 . Moreover, we set the probability for class 2 to 100% via y_desired_proba=1. from mlxtend.evaluate import create_counterfactual res = create_counterfactual(x_reference=x_ref, y_desired=2, model=clf, X_dataset=X, y_desired_proba=1., lammbda=1, # hyperparameter random_seed=123) print('Features of the 16th training example:', x_ref) print('Features of the countefactual:', res) print('Predictions for counterfactual:\\n') print('Predicted label:', clf.predict(res.reshape(1, -1))[0]) print('Predicted probas:', clf.predict_proba(res.reshape(1, -1))) Features of the 16th training example: [5.7 4.4 1.5 0.4] Features of the countefactual: [5.72271344 3.99169005 6.45305374 0.40000002] Predictions for counterfactual: Predicted label: 2 Predicted probas: [[1.41639932e-04 3.13292297e-01 6.86566063e-01]] As we can see above, the counterfactual is relatively similar to the original training example, i.e, only the 3rd feature has changed substantially (from 1.5 to 6.45). The predicted label has changed from class 0 t class 2. Interpretation-wise, this means increasing the petal length of a Iris-setosa flower may make it more similar to a Iris-virginica flower.","title":"Example 1 -- Simple Iris Example"},{"location":"user_guide/evaluate/create_counterfactual/#example-2-simple-iris-example-with-decision-regions-and-threshold-stopping-criterion","text":"This example is similar to Example 1; however, it is based on a 2D iris dataset containing only petal length and petal width features so that the results can be plotted via a decision region plot. from mlxtend.plotting import plot_decision_regions import matplotlib.pyplot as plt X, y = iris_data() X = X[:, 2:] clf = LogisticRegression() clf.fit(X, y) LogisticRegression() # Plotting decision regions ax = plot_decision_regions(X, y, clf=clf, legend=2) scatter_highlight_defaults = {'c': 'red', 'edgecolor': 'yellow', 'alpha': 1.0, 'linewidths': 2, 'marker': 'o', 's': 80} ax.scatter(*X[15], **scatter_highlight_defaults) plt.show() The big, highlighted point in the plot above shows the 16th training datapoint. The following code will create a counterfactual with the same settings as in Example 1: counterfact = create_counterfactual(x_reference=X[15], y_desired=2, model=clf, X_dataset=X, y_desired_proba=1.0, lammbda=1, random_seed=123) ax = plot_decision_regions(X, y, clf=clf, legend=2) ax.scatter(*counterfact, **scatter_highlight_defaults) plt.show() As we can see above, the counterfactual primarily moved along the x-axis (petal length) so that the prediction between the rerence point and the counterfactual changes from class 0 to class 2. The following plots are based on repeating this procedure with different lambda values: for i in [0.4, 0.5, 1.0, 5.0, 100]: counterfact = create_counterfactual(x_reference=X[15], y_desired=2, model=clf, X_dataset=X, y_desired_proba=1.0, lammbda=i, random_seed=123) ax = plot_decision_regions(X, y, clf=clf, legend=2) ax.scatter(*counterfact, **scatter_highlight_defaults) plt.show() As we can see, the stronger the \\lambda value, the more the first term in the loss L\\left(x, x^{\\prime}, y^{\\prime}, \\lambda\\right)=\\lambda \\cdot\\left(\\hat{f}\\left(x^{\\prime}\\right)-y^{\\prime}\\right)^{2}+d\\left(x, x^{\\prime}\\right). dominates. Applying Wachter et al.'s threshold concept, Optionally, as the authors recommend, you can repeat steps 2 and 3 by increasing \\lambda until a user-defined threshold \\epsilon is reached, i.e., while \\left|\\hat{f}\\left(x^{\\prime}\\right)-y^{\\prime}\\right|>\\epsilon : increase \\lambda we can define a user-defined threshold and implement it as follows: import numpy as np desired_class_2_proba = 1.0 for i in np.arange(0, 10000, 0.1): counterfact = create_counterfactual(x_reference=X[15], y_desired=2, model=clf, X_dataset=X, y_desired_proba=desired_class_2_proba, lammbda=i, random_seed=123) predicted_class_2_proba = clf.predict_proba(counterfact.reshape(1, -1))[0][2] if not i: print('Initial lambda:', i) print('Initial diff:', np.abs(predicted_class_2_proba - desired_class_2_proba)) if not np.abs(predicted_class_2_proba - desired_class_2_proba) > 0.3: break ax = plot_decision_regions(X, y, clf=clf, legend=2) ax.scatter(*counterfact, **scatter_highlight_defaults) print('Final lambda:', i) print('Final diff:', np.abs(predicted_class_2_proba - desired_class_2_proba)) plt.show() Initial lambda: 0.0 Initial diff: 0.9999998976132334 Final lambda: 1.1 Final diff: 0.2962621523225484","title":"Example 2 -- Simple Iris Example with Decision Regions and Threshold Stopping Criterion"},{"location":"user_guide/evaluate/create_counterfactual/#api","text":"create_counterfactual(x_reference, y_desired, model, X_dataset, y_desired_proba=None, lammbda=0.1, random_seed=None) Implementation of the counterfactual method by Wachter et al. References: Wachter, S., Mittelstadt, B., & Russell, C. (2017). Counterfactual explanations without opening the black box: Automated decisions and the GDPR. Harv. JL & Tech., 31, 841., https://arxiv.org/abs/1711.00399 Parameters x_reference : array-like, shape=[m_features] The data instance (training example) to be explained. y_desired : int The desired class label for x_reference . model : estimator A (scikit-learn) estimator implementing .predict() and/or predict_proba() . - If model supports predict_proba() , then this is used by default for the first loss term, (lambda * model.predict[_proba](x_counterfact) - y_desired[_proba])^2 - Otherwise, method will fall back to predict . X_dataset : array-like, shape=[n_examples, m_features] A (training) dataset for picking the initial counterfactual as initial value for starting the optimization procedure. y_desired_proba : float (default: None) A float within the range [0, 1] designating the desired class probability for y_desired . - If y_desired_proba=None (default), the first loss term is (lambda * model(x_counterfact) - y_desired)^2 where y_desired is a class label - If y_desired_proba is not None, the first loss term is (lambda * model(x_counterfact) - y_desired_proba)^2 lammbda : Weighting parameter for the first loss term, (lambda * model(x_counterfact) - y_desired[_proba])^2 random_seed : int (default=None) If int, random_seed is the seed used by the random number generator for selecting the inital counterfactual from X_dataset . ython","title":"API"},{"location":"user_guide/evaluate/feature_importance_permutation/","text":"feature_importance_permutation: Estimate feature importance via feature permutation. A function to estimate the feature importance of classifiers and regressors based on permutation importance . from mlxtend.evaluate import feature_importance_permutation Overview The permutation importance is an intuitive, model-agnostic method to estimate the feature importance for classifier and regression models. The approach is relatively simple and straight-forward: Take a model that was fit to the training dataset Estimate the predictive performance of the model on an independent dataset (e.g., validation dataset) and record it as the baseline performance For each feature i : randomly permute feature column i in the original dataset record the predictive performance of the model on the dataset with the permuted column compute the feature importance as the difference between the baseline performance (step 2) and the performance on the permuted dataset Permutation importance is generally considered as a relatively efficient technique that works well in practice [1], while a drawback is that the importance of correlated features may be overestimated [2]. References [1] Terence Parr, Kerem Turgutlu, Christopher Csiszar, and Jeremy Howard. Beware Default Random Forest Importances (https://parrt.cs.usfca.edu/doc/rf-importance/index.html) [2] Strobl, C., Boulesteix, A. L., Kneib, T., Augustin, T., & Zeileis, A. (2008). Conditional variable importance for random forests. BMC bioinformatics, 9(1), 307. Example 1 -- Feature Importance for Classifiers The following example illustrates the feature importance estimation via permutation importance based for classification models. import numpy as np import matplotlib.pyplot as plt from sklearn.svm import SVC from sklearn.model_selection import train_test_split from mlxtend.evaluate import feature_importance_permutation Generate a toy dataset from sklearn.datasets import make_classification from sklearn.ensemble import RandomForestClassifier # Build a classification task using 3 informative features X, y = make_classification(n_samples=10000, n_features=10, n_informative=3, n_redundant=0, n_repeated=0, n_classes=2, random_state=0, shuffle=False) X_train, X_test, y_train, y_test = train_test_split( X, y, test_size=0.3, random_state=1, stratify=y) Feature importance via random forest First, we compute the feature importance directly from the random forest via mean impurity decrease (described after the code section): forest = RandomForestClassifier(n_estimators=250, random_state=0) forest.fit(X_train, y_train) print('Training accuracy:', np.mean(forest.predict(X_train) == y_train)*100) print('Test accuracy:', np.mean(forest.predict(X_test) == y_test)*100) importance_vals = forest.feature_importances_ print(importance_vals) Training accuracy: 100.0 Test accuracy: 95.06666666666666 [0.283357 0.30846795 0.24204291 0.02229767 0.02364941 0.02390578 0.02501543 0.0234225 0.02370816 0.0241332 ] There are several strategies for computing the feature importance in random forest. The method implemented in scikit-learn (used in the next code example) is based on the Breiman and Friedman's CART (Breiman, Friedman, \"Classification and regression trees\", 1984), the so-called mean impurity decrease . Here, the importance value of a features is computed by averaging the impurity decrease for that feature, when splitting a parent node into two child nodes, across all the trees in the ensemble. Note that the impurity decrease values are weighted by the number of samples that are in the respective nodes. This process is repeated for all features in the dataset, and the feature importance values are then normalized so that they sum up to 1. In CART, the authors also note that this fast way of computing feature importance values is relatively consistent with the permutation importance. Next, let's visualize the feature importance values from the random forest including a measure of the mean impurity decrease variability (here: standard deviation): std = np.std([tree.feature_importances_ for tree in forest.estimators_], axis=0) indices = np.argsort(importance_vals)[::-1] # Plot the feature importances of the forest plt.figure() plt.title(\"Random Forest feature importance\") plt.bar(range(X.shape[1]), importance_vals[indices], yerr=std[indices], align=\"center\") plt.xticks(range(X.shape[1]), indices) plt.xlim([-1, X.shape[1]]) plt.ylim([0, 0.5]) plt.show() As we can see, the features 1, 0, and 2 are estimated to be the most informative ones for the random forest classier. Next, let's compute the feature importance via the permutation importance approach. Permutation Importance imp_vals, _ = feature_importance_permutation( predict_method=forest.predict, X=X_test, y=y_test, metric='accuracy', num_rounds=1, seed=1) imp_vals array([ 0.26833333, 0.26733333, 0.261 , -0.002 , -0.00033333, 0.00066667, 0.00233333, 0.00066667, 0.00066667, -0.00233333]) Note that the feature_importance_permutation returns two arrays. The first array (here: imp_vals ) contains the actual importance values we are interested in. If num_rounds > 1 , the permutation is repeated multiple times (with different random seeds), and in this case the first array contains the average value of the importance computed from the different runs. The second array (here, assigned to _ , because we are not using it) then contains all individual values from these runs (more about that later). Now, let's also visualize the importance values in a barplot: indices = np.argsort(imp_vals)[::-1] plt.figure() plt.title(\"Random Forest feature importance via permutation importance\") plt.bar(range(X.shape[1]), imp_vals[indices]) plt.xticks(range(X.shape[1]), indices) plt.xlim([-1, X.shape[1]]) plt.ylim([0, 0.5]) plt.show() As we can see, also here, features 1, 0, and 2 are predicted to be the most important ones, which is consistent with the feature importance values that we computed via the mean impurity decrease method earlier. (Note that in the context of random forests, the feature importance via permutation importance is typically computed using the out-of-bag samples of a random forest, whereas in this implementation, an independent dataset is used.) Previously, it was mentioned that the permutation is repeated multiple times if num_rounds > 1 . In this case, the second array returned by the feature_importance_permutation contains the importance values for these individual runs (the array has shape [num_features, num_rounds), which we can use to compute some sort of variability between these runs. imp_vals, imp_all = feature_importance_permutation( predict_method=forest.predict, X=X_test, y=y_test, metric='accuracy', num_rounds=10, seed=1) std = np.std(imp_all, axis=1) indices = np.argsort(imp_vals)[::-1] plt.figure() plt.title(\"Random Forest feature importance via permutation importance w. std. dev.\") plt.bar(range(X.shape[1]), imp_vals[indices], yerr=std[indices]) plt.xticks(range(X.shape[1]), indices) plt.xlim([-1, X.shape[1]]) plt.show() It shall be noted that the feature importance values do not sum up to one, since they are not normalized (you can normalize them if you'd like, by dividing these by the sum of importance values). Here, the main point is to look at the importance values relative to each other and not to over-interpret the absolute values. Support Vector Machines While the permutation importance approach yields results that are generally consistent with the mean impurity decrease feature importance values from a random forest, it's a method that is model-agnostic and can be used with any kind of classifier or regressor. The example below applies the feature_importance_permutation function to a support vector machine: from sklearn.svm import SVC svm = SVC(C=1.0, kernel='rbf') svm.fit(X_train, y_train) print('Training accuracy', np.mean(svm.predict(X_train) == y_train)*100) print('Test accuracy', np.mean(svm.predict(X_test) == y_test)*100) Training accuracy 94.87142857142857 Test accuracy 94.89999999999999 imp_vals, imp_all = feature_importance_permutation( predict_method=svm.predict, X=X_test, y=y_test, metric='accuracy', num_rounds=10, seed=1) std = np.std(imp_all, axis=1) indices = np.argsort(imp_vals)[::-1] plt.figure() plt.title(\"SVM feature importance via permutation importance\") plt.bar(range(X.shape[1]), imp_vals[indices], yerr=std[indices]) plt.xticks(range(X.shape[1]), indices) plt.xlim([-1, X.shape[1]]) plt.show() Example 2 -- Feature Importance for Regressors import numpy as np import matplotlib.pyplot as plt from mlxtend.evaluate import feature_importance_permutation from sklearn.model_selection import train_test_split from sklearn.datasets import make_regression from sklearn.svm import SVR X, y = make_regression(n_samples=1000, n_features=5, n_informative=2, n_targets=1, random_state=123, shuffle=False) X_train, X_test, y_train, y_test = train_test_split( X, y, test_size=0.3, random_state=123) svm = SVR(kernel='rbf') svm.fit(X_train, y_train) imp_vals, _ = feature_importance_permutation( predict_method=svm.predict, X=X_test, y=y_test, metric='r2', num_rounds=1, seed=1) imp_vals array([ 0.43309137, 0.22058866, 0.00148447, 0.01613953, -0.00529505]) plt.figure() plt.bar(range(X.shape[1]), imp_vals) plt.xticks(range(X.shape[1])) plt.xlim([-1, X.shape[1]]) plt.ylim([0, 0.5]) plt.show() Example 3 -- Feature Importance With One-Hot-Encoded Features Upon one-hot encoding, a feature variable with 10 distinct categories will be split into 10 new feature columns (or 9, if you drop a redundant column). If we want to treat each new feature column as an individual feature variable, we can use the feature_importance_permutation as usual. This is illustrated in the example below. Preparing the Dataset Here, we look at a dataset that consists of one categorical feature ( 'categorical' ) and 3 numerical features ( 'measurement1' , 'measurement2' , and 'measurement3' ). import pandas as pd df_data = pd.read_csv('https://gist.githubusercontent.com/rasbt/b99bf69079bc0d601eeae8a49248d358/raw/a114be9801647ec5460089f3a9576713dabf5f1f/onehot-numeric-mixed-data.csv') df_data.head() .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } categorical measurement1 measurement2 label measurement3 0 F 1.428571 2.721313 0 2.089 1 R 0.685939 0.982976 0 0.637 2 P 1.055817 0.624210 0 0.226 3 S 0.995956 0.321101 0 0.138 4 R 1.376773 1.578309 0 0.478 from sklearn.model_selection import train_test_split df_X = df_data[['measurement1', 'measurement2', 'measurement3', 'categorical']] df_y = df_data['label'] df_X_train, df_X_test, df_y_train, df_y_test = train_test_split( df_X, df_y, test_size=0.33, random_state=42, stratify=df_y) Here, we do the one-hot encoding on the categorical feature and merge it back with the numerical columns: from sklearn.preprocessing import OneHotEncoder import numpy as np ohe = OneHotEncoder(drop='first') ohe.fit(df_X_train[['categorical']]) df_X_train_ohe = df_X_train.drop(columns=['categorical']) df_X_test_ohe = df_X_test.drop(columns=['categorical']) ohe_train = np.asarray(ohe.transform(df_X_train[['categorical']]).todense()) ohe_test = np.asarray(ohe.transform(df_X_test[['categorical']]).todense()) X_train_ohe = np.hstack((df_X_train_ohe.values, ohe_train)) X_test_ohe = np.hstack((df_X_test_ohe.values, ohe_test)) # look at first 3 rows print(X_train_ohe[:3]) [[0.65747208 0.95105388 0.36 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 1. 0. 0. 0. 0. 0. ] [1.17503636 1.01094494 0.653 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 1. 0. 0. 0. 0. ] [1.25516647 0.67575824 0.176 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 1. 0. 0. 0. 0. 0. 0. ]] Fitting a Baseline Model for Analysis from sklearn.neural_network import MLPClassifier from sklearn.preprocessing import StandardScaler from sklearn.pipeline import make_pipeline from sklearn.model_selection import GridSearchCV pipe = make_pipeline(StandardScaler(), MLPClassifier(max_iter=10000, random_state=123)) params = { 'mlpclassifier__hidden_layer_sizes': [(30, 20, 10), (20, 10), (20,), (10,)], 'mlpclassifier__activation': ['tanh', 'relu'], 'mlpclassifier__solver': ['sgd'], 'mlpclassifier__alpha': [0.0001], 'mlpclassifier__learning_rate': ['adaptive'], } gs = GridSearchCV(estimator=pipe, param_grid=params, scoring='accuracy', refit=True, n_jobs=-1, cv=10) gs = gs.fit(X_train_ohe, df_y_train.values) model = gs.best_estimator_ Regular Permutation Importance Here, we compute the feature importance the regular way were each one-hot encoded feature is treated as an individiual variable. from mlxtend.evaluate import feature_importance_permutation imp_vals, imp_all = feature_importance_permutation( predict_method=model.predict, X=X_test_ohe, y=df_y_test.values, metric='accuracy', num_rounds=50, seed=1) feat_names_with_ohe = ['measurement1', 'measurement2', 'measurement3'] \\ + [f'categorical_ohe_{i}' for i in range(2, 20)] %matplotlib inline import matplotlib.pyplot as plt std = np.std(imp_all, axis=1) indices = np.argsort(imp_vals)[::-1] plt.figure() #plt.title(\"Feature importance via permutation importance w. std. dev.\") plt.bar(range(len(feat_names_with_ohe)), imp_vals[indices], yerr=std[indices]) plt.xticks(range(len(feat_names_with_ohe)), np.array(feat_names_with_ohe)[indices], rotation=90) plt.xlim([-1, len(feat_names_with_ohe)]) plt.show() However, notice that if we have a lot of categorical values, the feature importance of the individual binary features (after one hot encoding) are now hard to interpret. In certain cases, it would be desireable to treat the one-hot encoded binary features as a single variable in feature permutation importance evaluation. We can achieve this using feature groups. With Feature Groups In the example below, all the one-hot encoded variables are treated as a feature group. This means, they are are all shuffled and analyzed as a single feature inside the feature permutation importance analysis. feature_groups = [0, 1, 2, range(3, 21)] imp_vals, imp_all = feature_importance_permutation( predict_method=model.predict, X=X_test_ohe, y=df_y_test.values, metric='accuracy', num_rounds=50, feature_groups=feature_groups, seed=1) feature_names = ['measurement1', 'measurement2', 'measurement3', 'categorical'] std = np.std(imp_all, axis=1) indices = np.argsort(imp_vals)[::-1] plt.figure() plt.bar(range(len(feature_names)), imp_vals[indices], yerr=std[indices]) plt.xticks(range(len(feature_names)), np.array(feature_names)[indices], rotation=90) plt.xlim([-1, len(feature_names)]) plt.show() API feature_importance_permutation(X, y, predict_method, metric, num_rounds=1, feature_groups=None, seed=None) Feature importance imputation via permutation importance Parameters X : NumPy array, shape = [n_samples, n_features] Dataset, where n_samples is the number of samples and n_features is the number of features. y : NumPy array, shape = [n_samples] Target values. predict_method : prediction function A callable function that predicts the target values from X. metric : str, callable The metric for evaluating the feature importance through permutation. By default, the strings 'accuracy' is recommended for classifiers and the string 'r2' is recommended for regressors. Optionally, a custom scoring function (e.g., metric=scoring_func ) that accepts two arguments, y_true and y_pred, which have similar shape to the y array. num_rounds : int (default=1) Number of rounds the feature columns are permuted to compute the permutation importance. feature_groups : list or None (default=None) Optional argument for treating certain features as a group. For example [1, 2, [3, 4, 5]] , which can be useful for interpretability, for example, if features 3, 4, 5 are one-hot encoded features. seed : int or None (default=None) Random seed for permuting the feature columns. Returns mean_importance_vals, all_importance_vals : NumPy arrays. The first array, mean_importance_vals has shape [n_features, ] and contains the importance values for all features. The shape of the second array is [n_features, num_rounds] and contains the feature importance for each repetition. If num_rounds=1, it contains the same values as the first array, mean_importance_vals. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/feature_importance_permutation/ ython ython","title":"feature_importance_permutation: Estimate feature importance via feature permutation."},{"location":"user_guide/evaluate/feature_importance_permutation/#feature_importance_permutation-estimate-feature-importance-via-feature-permutation","text":"A function to estimate the feature importance of classifiers and regressors based on permutation importance . from mlxtend.evaluate import feature_importance_permutation","title":"feature_importance_permutation: Estimate feature importance via feature permutation."},{"location":"user_guide/evaluate/feature_importance_permutation/#overview","text":"The permutation importance is an intuitive, model-agnostic method to estimate the feature importance for classifier and regression models. The approach is relatively simple and straight-forward: Take a model that was fit to the training dataset Estimate the predictive performance of the model on an independent dataset (e.g., validation dataset) and record it as the baseline performance For each feature i : randomly permute feature column i in the original dataset record the predictive performance of the model on the dataset with the permuted column compute the feature importance as the difference between the baseline performance (step 2) and the performance on the permuted dataset Permutation importance is generally considered as a relatively efficient technique that works well in practice [1], while a drawback is that the importance of correlated features may be overestimated [2].","title":"Overview"},{"location":"user_guide/evaluate/feature_importance_permutation/#references","text":"[1] Terence Parr, Kerem Turgutlu, Christopher Csiszar, and Jeremy Howard. Beware Default Random Forest Importances (https://parrt.cs.usfca.edu/doc/rf-importance/index.html) [2] Strobl, C., Boulesteix, A. L., Kneib, T., Augustin, T., & Zeileis, A. (2008). Conditional variable importance for random forests. BMC bioinformatics, 9(1), 307.","title":"References"},{"location":"user_guide/evaluate/feature_importance_permutation/#example-1-feature-importance-for-classifiers","text":"The following example illustrates the feature importance estimation via permutation importance based for classification models. import numpy as np import matplotlib.pyplot as plt from sklearn.svm import SVC from sklearn.model_selection import train_test_split from mlxtend.evaluate import feature_importance_permutation","title":"Example 1 -- Feature Importance for Classifiers"},{"location":"user_guide/evaluate/feature_importance_permutation/#generate-a-toy-dataset","text":"from sklearn.datasets import make_classification from sklearn.ensemble import RandomForestClassifier # Build a classification task using 3 informative features X, y = make_classification(n_samples=10000, n_features=10, n_informative=3, n_redundant=0, n_repeated=0, n_classes=2, random_state=0, shuffle=False) X_train, X_test, y_train, y_test = train_test_split( X, y, test_size=0.3, random_state=1, stratify=y)","title":"Generate a toy dataset"},{"location":"user_guide/evaluate/feature_importance_permutation/#feature-importance-via-random-forest","text":"First, we compute the feature importance directly from the random forest via mean impurity decrease (described after the code section): forest = RandomForestClassifier(n_estimators=250, random_state=0) forest.fit(X_train, y_train) print('Training accuracy:', np.mean(forest.predict(X_train) == y_train)*100) print('Test accuracy:', np.mean(forest.predict(X_test) == y_test)*100) importance_vals = forest.feature_importances_ print(importance_vals) Training accuracy: 100.0 Test accuracy: 95.06666666666666 [0.283357 0.30846795 0.24204291 0.02229767 0.02364941 0.02390578 0.02501543 0.0234225 0.02370816 0.0241332 ] There are several strategies for computing the feature importance in random forest. The method implemented in scikit-learn (used in the next code example) is based on the Breiman and Friedman's CART (Breiman, Friedman, \"Classification and regression trees\", 1984), the so-called mean impurity decrease . Here, the importance value of a features is computed by averaging the impurity decrease for that feature, when splitting a parent node into two child nodes, across all the trees in the ensemble. Note that the impurity decrease values are weighted by the number of samples that are in the respective nodes. This process is repeated for all features in the dataset, and the feature importance values are then normalized so that they sum up to 1. In CART, the authors also note that this fast way of computing feature importance values is relatively consistent with the permutation importance. Next, let's visualize the feature importance values from the random forest including a measure of the mean impurity decrease variability (here: standard deviation): std = np.std([tree.feature_importances_ for tree in forest.estimators_], axis=0) indices = np.argsort(importance_vals)[::-1] # Plot the feature importances of the forest plt.figure() plt.title(\"Random Forest feature importance\") plt.bar(range(X.shape[1]), importance_vals[indices], yerr=std[indices], align=\"center\") plt.xticks(range(X.shape[1]), indices) plt.xlim([-1, X.shape[1]]) plt.ylim([0, 0.5]) plt.show() As we can see, the features 1, 0, and 2 are estimated to be the most informative ones for the random forest classier. Next, let's compute the feature importance via the permutation importance approach.","title":"Feature importance via random forest"},{"location":"user_guide/evaluate/feature_importance_permutation/#permutation-importance","text":"imp_vals, _ = feature_importance_permutation( predict_method=forest.predict, X=X_test, y=y_test, metric='accuracy', num_rounds=1, seed=1) imp_vals array([ 0.26833333, 0.26733333, 0.261 , -0.002 , -0.00033333, 0.00066667, 0.00233333, 0.00066667, 0.00066667, -0.00233333]) Note that the feature_importance_permutation returns two arrays. The first array (here: imp_vals ) contains the actual importance values we are interested in. If num_rounds > 1 , the permutation is repeated multiple times (with different random seeds), and in this case the first array contains the average value of the importance computed from the different runs. The second array (here, assigned to _ , because we are not using it) then contains all individual values from these runs (more about that later). Now, let's also visualize the importance values in a barplot: indices = np.argsort(imp_vals)[::-1] plt.figure() plt.title(\"Random Forest feature importance via permutation importance\") plt.bar(range(X.shape[1]), imp_vals[indices]) plt.xticks(range(X.shape[1]), indices) plt.xlim([-1, X.shape[1]]) plt.ylim([0, 0.5]) plt.show() As we can see, also here, features 1, 0, and 2 are predicted to be the most important ones, which is consistent with the feature importance values that we computed via the mean impurity decrease method earlier. (Note that in the context of random forests, the feature importance via permutation importance is typically computed using the out-of-bag samples of a random forest, whereas in this implementation, an independent dataset is used.) Previously, it was mentioned that the permutation is repeated multiple times if num_rounds > 1 . In this case, the second array returned by the feature_importance_permutation contains the importance values for these individual runs (the array has shape [num_features, num_rounds), which we can use to compute some sort of variability between these runs. imp_vals, imp_all = feature_importance_permutation( predict_method=forest.predict, X=X_test, y=y_test, metric='accuracy', num_rounds=10, seed=1) std = np.std(imp_all, axis=1) indices = np.argsort(imp_vals)[::-1] plt.figure() plt.title(\"Random Forest feature importance via permutation importance w. std. dev.\") plt.bar(range(X.shape[1]), imp_vals[indices], yerr=std[indices]) plt.xticks(range(X.shape[1]), indices) plt.xlim([-1, X.shape[1]]) plt.show() It shall be noted that the feature importance values do not sum up to one, since they are not normalized (you can normalize them if you'd like, by dividing these by the sum of importance values). Here, the main point is to look at the importance values relative to each other and not to over-interpret the absolute values.","title":"Permutation Importance"},{"location":"user_guide/evaluate/feature_importance_permutation/#support-vector-machines","text":"While the permutation importance approach yields results that are generally consistent with the mean impurity decrease feature importance values from a random forest, it's a method that is model-agnostic and can be used with any kind of classifier or regressor. The example below applies the feature_importance_permutation function to a support vector machine: from sklearn.svm import SVC svm = SVC(C=1.0, kernel='rbf') svm.fit(X_train, y_train) print('Training accuracy', np.mean(svm.predict(X_train) == y_train)*100) print('Test accuracy', np.mean(svm.predict(X_test) == y_test)*100) Training accuracy 94.87142857142857 Test accuracy 94.89999999999999 imp_vals, imp_all = feature_importance_permutation( predict_method=svm.predict, X=X_test, y=y_test, metric='accuracy', num_rounds=10, seed=1) std = np.std(imp_all, axis=1) indices = np.argsort(imp_vals)[::-1] plt.figure() plt.title(\"SVM feature importance via permutation importance\") plt.bar(range(X.shape[1]), imp_vals[indices], yerr=std[indices]) plt.xticks(range(X.shape[1]), indices) plt.xlim([-1, X.shape[1]]) plt.show()","title":"Support Vector Machines"},{"location":"user_guide/evaluate/feature_importance_permutation/#example-2-feature-importance-for-regressors","text":"import numpy as np import matplotlib.pyplot as plt from mlxtend.evaluate import feature_importance_permutation from sklearn.model_selection import train_test_split from sklearn.datasets import make_regression from sklearn.svm import SVR X, y = make_regression(n_samples=1000, n_features=5, n_informative=2, n_targets=1, random_state=123, shuffle=False) X_train, X_test, y_train, y_test = train_test_split( X, y, test_size=0.3, random_state=123) svm = SVR(kernel='rbf') svm.fit(X_train, y_train) imp_vals, _ = feature_importance_permutation( predict_method=svm.predict, X=X_test, y=y_test, metric='r2', num_rounds=1, seed=1) imp_vals array([ 0.43309137, 0.22058866, 0.00148447, 0.01613953, -0.00529505]) plt.figure() plt.bar(range(X.shape[1]), imp_vals) plt.xticks(range(X.shape[1])) plt.xlim([-1, X.shape[1]]) plt.ylim([0, 0.5]) plt.show()","title":"Example 2 -- Feature Importance for Regressors"},{"location":"user_guide/evaluate/feature_importance_permutation/#example-3-feature-importance-with-one-hot-encoded-features","text":"Upon one-hot encoding, a feature variable with 10 distinct categories will be split into 10 new feature columns (or 9, if you drop a redundant column). If we want to treat each new feature column as an individual feature variable, we can use the feature_importance_permutation as usual. This is illustrated in the example below.","title":"Example 3 -- Feature Importance With One-Hot-Encoded Features"},{"location":"user_guide/evaluate/feature_importance_permutation/#preparing-the-dataset","text":"Here, we look at a dataset that consists of one categorical feature ( 'categorical' ) and 3 numerical features ( 'measurement1' , 'measurement2' , and 'measurement3' ). import pandas as pd df_data = pd.read_csv('https://gist.githubusercontent.com/rasbt/b99bf69079bc0d601eeae8a49248d358/raw/a114be9801647ec5460089f3a9576713dabf5f1f/onehot-numeric-mixed-data.csv') df_data.head() .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } categorical measurement1 measurement2 label measurement3 0 F 1.428571 2.721313 0 2.089 1 R 0.685939 0.982976 0 0.637 2 P 1.055817 0.624210 0 0.226 3 S 0.995956 0.321101 0 0.138 4 R 1.376773 1.578309 0 0.478 from sklearn.model_selection import train_test_split df_X = df_data[['measurement1', 'measurement2', 'measurement3', 'categorical']] df_y = df_data['label'] df_X_train, df_X_test, df_y_train, df_y_test = train_test_split( df_X, df_y, test_size=0.33, random_state=42, stratify=df_y) Here, we do the one-hot encoding on the categorical feature and merge it back with the numerical columns: from sklearn.preprocessing import OneHotEncoder import numpy as np ohe = OneHotEncoder(drop='first') ohe.fit(df_X_train[['categorical']]) df_X_train_ohe = df_X_train.drop(columns=['categorical']) df_X_test_ohe = df_X_test.drop(columns=['categorical']) ohe_train = np.asarray(ohe.transform(df_X_train[['categorical']]).todense()) ohe_test = np.asarray(ohe.transform(df_X_test[['categorical']]).todense()) X_train_ohe = np.hstack((df_X_train_ohe.values, ohe_train)) X_test_ohe = np.hstack((df_X_test_ohe.values, ohe_test)) # look at first 3 rows print(X_train_ohe[:3]) [[0.65747208 0.95105388 0.36 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 1. 0. 0. 0. 0. 0. ] [1.17503636 1.01094494 0.653 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 1. 0. 0. 0. 0. ] [1.25516647 0.67575824 0.176 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 1. 0. 0. 0. 0. 0. 0. ]]","title":"Preparing the Dataset"},{"location":"user_guide/evaluate/feature_importance_permutation/#fitting-a-baseline-model-for-analysis","text":"from sklearn.neural_network import MLPClassifier from sklearn.preprocessing import StandardScaler from sklearn.pipeline import make_pipeline from sklearn.model_selection import GridSearchCV pipe = make_pipeline(StandardScaler(), MLPClassifier(max_iter=10000, random_state=123)) params = { 'mlpclassifier__hidden_layer_sizes': [(30, 20, 10), (20, 10), (20,), (10,)], 'mlpclassifier__activation': ['tanh', 'relu'], 'mlpclassifier__solver': ['sgd'], 'mlpclassifier__alpha': [0.0001], 'mlpclassifier__learning_rate': ['adaptive'], } gs = GridSearchCV(estimator=pipe, param_grid=params, scoring='accuracy', refit=True, n_jobs=-1, cv=10) gs = gs.fit(X_train_ohe, df_y_train.values) model = gs.best_estimator_","title":"Fitting a Baseline Model for Analysis"},{"location":"user_guide/evaluate/feature_importance_permutation/#regular-permutation-importance","text":"Here, we compute the feature importance the regular way were each one-hot encoded feature is treated as an individiual variable. from mlxtend.evaluate import feature_importance_permutation imp_vals, imp_all = feature_importance_permutation( predict_method=model.predict, X=X_test_ohe, y=df_y_test.values, metric='accuracy', num_rounds=50, seed=1) feat_names_with_ohe = ['measurement1', 'measurement2', 'measurement3'] \\ + [f'categorical_ohe_{i}' for i in range(2, 20)] %matplotlib inline import matplotlib.pyplot as plt std = np.std(imp_all, axis=1) indices = np.argsort(imp_vals)[::-1] plt.figure() #plt.title(\"Feature importance via permutation importance w. std. dev.\") plt.bar(range(len(feat_names_with_ohe)), imp_vals[indices], yerr=std[indices]) plt.xticks(range(len(feat_names_with_ohe)), np.array(feat_names_with_ohe)[indices], rotation=90) plt.xlim([-1, len(feat_names_with_ohe)]) plt.show() However, notice that if we have a lot of categorical values, the feature importance of the individual binary features (after one hot encoding) are now hard to interpret. In certain cases, it would be desireable to treat the one-hot encoded binary features as a single variable in feature permutation importance evaluation. We can achieve this using feature groups.","title":"Regular Permutation Importance"},{"location":"user_guide/evaluate/feature_importance_permutation/#with-feature-groups","text":"In the example below, all the one-hot encoded variables are treated as a feature group. This means, they are are all shuffled and analyzed as a single feature inside the feature permutation importance analysis. feature_groups = [0, 1, 2, range(3, 21)] imp_vals, imp_all = feature_importance_permutation( predict_method=model.predict, X=X_test_ohe, y=df_y_test.values, metric='accuracy', num_rounds=50, feature_groups=feature_groups, seed=1) feature_names = ['measurement1', 'measurement2', 'measurement3', 'categorical'] std = np.std(imp_all, axis=1) indices = np.argsort(imp_vals)[::-1] plt.figure() plt.bar(range(len(feature_names)), imp_vals[indices], yerr=std[indices]) plt.xticks(range(len(feature_names)), np.array(feature_names)[indices], rotation=90) plt.xlim([-1, len(feature_names)]) plt.show()","title":"With Feature Groups"},{"location":"user_guide/evaluate/feature_importance_permutation/#api","text":"feature_importance_permutation(X, y, predict_method, metric, num_rounds=1, feature_groups=None, seed=None) Feature importance imputation via permutation importance Parameters X : NumPy array, shape = [n_samples, n_features] Dataset, where n_samples is the number of samples and n_features is the number of features. y : NumPy array, shape = [n_samples] Target values. predict_method : prediction function A callable function that predicts the target values from X. metric : str, callable The metric for evaluating the feature importance through permutation. By default, the strings 'accuracy' is recommended for classifiers and the string 'r2' is recommended for regressors. Optionally, a custom scoring function (e.g., metric=scoring_func ) that accepts two arguments, y_true and y_pred, which have similar shape to the y array. num_rounds : int (default=1) Number of rounds the feature columns are permuted to compute the permutation importance. feature_groups : list or None (default=None) Optional argument for treating certain features as a group. For example [1, 2, [3, 4, 5]] , which can be useful for interpretability, for example, if features 3, 4, 5 are one-hot encoded features. seed : int or None (default=None) Random seed for permuting the feature columns. Returns mean_importance_vals, all_importance_vals : NumPy arrays. The first array, mean_importance_vals has shape [n_features, ] and contains the importance values for all features. The shape of the second array is [n_features, num_rounds] and contains the feature importance for each repetition. If num_rounds=1, it contains the same values as the first array, mean_importance_vals. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/feature_importance_permutation/ ython ython","title":"API"},{"location":"user_guide/evaluate/ftest/","text":"ftest: F-test for classifier comparisons F-test for comparing the performance of multiple classifiers. from mlxtend.evaluate import ftest Overview In the context of evaluating machine learning models, the F-test by George W. Snedecor [1] can be regarded as analogous to Cochran's Q test that can be applied to evaluate multiple classifiers (i.e., whether their accuracies estimated on a test set differ) as described by Looney [2][3]. More formally, assume the task to test the null hypothesis that there is no difference between the classification accuracies [1]: p_i: H_0 = p_1 = p_2 = \\cdots = p_L. Let \\{C_1, \\dots , C_M\\} be a set of classifiers which have all been tested on the same dataset. If the M classifiers do not perform differently, then the F statistic is distributed according to an F distribution with (M-1) and (M-1)\\times n degrees of freedom, where n is the number of examples in the test set. The calculation of the F statistic consists of several components, which are listed below (adopted from [2]). We start by defining ACC_{avg} as the average of the accuracies of the different models ACC_{avg} = \\frac{1}{M}\\sum_{j=1}^M ACC_j. The sum of squares of the classifiers is then computed as SSA = n \\sum_{j=1}^{M} (G_j)^2 -n \\cdot M \\cdot ACC_{avg}, where G_j is the proportion of the n examples classified correctly by classifier j . The sum of squares for the objects is calculated as follows: SSB= \\frac{1}{M} \\sum_{j=1}^n (M_j)^2 - M\\cdot n \\cdot ACC_{avg}^2. Here, M_j is the number of classifiers out of M that correctly classified object \\mathbf{x}_j \\in \\mathbf{X}_{n} , where \\mathbf{X}_{n} = \\{\\mathbf{x}_1, ... \\mathbf{x}_{n}\\} is the test dataset on which the classifiers are tested on. Finally, we compute the total sum of squares, SST = M\\cdot n \\cdot ACC_{avg} (1 - ACC_{avg}), so that we then can compute the sum of squares for the classification--object interaction: SSAB = SST - SSA - SSB. To compute the F statistic, we next compute the mean SSA and mean SSAB values: MSA = \\frac{SSA}{M-1}, and MSAB = \\frac{SSAB}{(M-1) (n-1)}. From the MSA and MSAB, we can then calculate the F-value as F = \\frac{MSA}{MSAB}. After computing the F-value, we can then look up the p-value from a F-distribution table for the corresponding degrees of freedom or obtain it computationally from a cumulative F-distribution function. In practice, if we successfully rejected the null hypothesis at a previously chosen significance threshold, we could perform multiple post hoc pair-wise tests -- for example, McNemar tests with a Bonferroni correction -- to determine which pairs have different population proportions. References [1] Snedecor, George W. and Cochran, William G. (1989), Statistical Methods, Eighth Edition, Iowa State University Press. [2] Looney, Stephen W. \"A statistical technique for comparing the accuracies of several classifiers.\" Pattern Recognition Letters 8, no. 1 (1988): 5-9. [3] Kuncheva, Ludmila I. Combining pattern classifiers: methods and algorithms. John Wiley & Sons, 2004. Example 1 - F-test import numpy as np from mlxtend.evaluate import ftest ## Dataset: # ground truth labels of the test dataset: y_true = np.array([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]) # predictions by 3 classifiers (`y_model_1`, `y_model_2`, and `y_model_3`): y_model_1 = np.array([1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]) y_model_2 = np.array([1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]) y_model_3 = np.array([1, 1, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1]) Assuming a significance level \\alpha=0.05 , we can conduct Cochran's Q test as follows, to test the null hypothesis there is no difference between the classification accuracies, p_i: H_0 = p_1 = p_2 = \\cdots = p_L : f, p_value = ftest(y_true, y_model_1, y_model_2, y_model_3) print('F: %.3f' % f) print('p-value: %.3f' % p_value) F: 3.873 p-value: 0.022 Since the p-value is smaller than \\alpha , we can reject the null hypothesis and conclude that there is a difference between the classification accuracies. As mentioned in the introduction earlier, we could now perform multiple post hoc pair-wise tests -- for example, McNemar tests with a Bonferroni correction -- to determine which pairs have different population proportions. API ftest(y_target, y_model_predictions)* F-Test test to compare 2 or more models. Parameters y_target : array-like, shape=[n_samples] True class labels as 1D NumPy array. *y_model_predictions : array-likes, shape=[n_samples] Variable number of 2 or more arrays that contain the predicted class labels from models as 1D NumPy array. Returns f, p : float or None, float Returns the F-value and the p-value Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/ftest/","title":"ftest: F-test for classifier comparisons"},{"location":"user_guide/evaluate/ftest/#ftest-f-test-for-classifier-comparisons","text":"F-test for comparing the performance of multiple classifiers. from mlxtend.evaluate import ftest","title":"ftest: F-test for classifier comparisons"},{"location":"user_guide/evaluate/ftest/#overview","text":"In the context of evaluating machine learning models, the F-test by George W. Snedecor [1] can be regarded as analogous to Cochran's Q test that can be applied to evaluate multiple classifiers (i.e., whether their accuracies estimated on a test set differ) as described by Looney [2][3]. More formally, assume the task to test the null hypothesis that there is no difference between the classification accuracies [1]: p_i: H_0 = p_1 = p_2 = \\cdots = p_L. Let \\{C_1, \\dots , C_M\\} be a set of classifiers which have all been tested on the same dataset. If the M classifiers do not perform differently, then the F statistic is distributed according to an F distribution with (M-1) and (M-1)\\times n degrees of freedom, where n is the number of examples in the test set. The calculation of the F statistic consists of several components, which are listed below (adopted from [2]). We start by defining ACC_{avg} as the average of the accuracies of the different models ACC_{avg} = \\frac{1}{M}\\sum_{j=1}^M ACC_j. The sum of squares of the classifiers is then computed as SSA = n \\sum_{j=1}^{M} (G_j)^2 -n \\cdot M \\cdot ACC_{avg}, where G_j is the proportion of the n examples classified correctly by classifier j . The sum of squares for the objects is calculated as follows: SSB= \\frac{1}{M} \\sum_{j=1}^n (M_j)^2 - M\\cdot n \\cdot ACC_{avg}^2. Here, M_j is the number of classifiers out of M that correctly classified object \\mathbf{x}_j \\in \\mathbf{X}_{n} , where \\mathbf{X}_{n} = \\{\\mathbf{x}_1, ... \\mathbf{x}_{n}\\} is the test dataset on which the classifiers are tested on. Finally, we compute the total sum of squares, SST = M\\cdot n \\cdot ACC_{avg} (1 - ACC_{avg}), so that we then can compute the sum of squares for the classification--object interaction: SSAB = SST - SSA - SSB. To compute the F statistic, we next compute the mean SSA and mean SSAB values: MSA = \\frac{SSA}{M-1}, and MSAB = \\frac{SSAB}{(M-1) (n-1)}. From the MSA and MSAB, we can then calculate the F-value as F = \\frac{MSA}{MSAB}. After computing the F-value, we can then look up the p-value from a F-distribution table for the corresponding degrees of freedom or obtain it computationally from a cumulative F-distribution function. In practice, if we successfully rejected the null hypothesis at a previously chosen significance threshold, we could perform multiple post hoc pair-wise tests -- for example, McNemar tests with a Bonferroni correction -- to determine which pairs have different population proportions.","title":"Overview"},{"location":"user_guide/evaluate/ftest/#references","text":"[1] Snedecor, George W. and Cochran, William G. (1989), Statistical Methods, Eighth Edition, Iowa State University Press. [2] Looney, Stephen W. \"A statistical technique for comparing the accuracies of several classifiers.\" Pattern Recognition Letters 8, no. 1 (1988): 5-9. [3] Kuncheva, Ludmila I. Combining pattern classifiers: methods and algorithms. John Wiley & Sons, 2004.","title":"References"},{"location":"user_guide/evaluate/ftest/#example-1-f-test","text":"import numpy as np from mlxtend.evaluate import ftest ## Dataset: # ground truth labels of the test dataset: y_true = np.array([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]) # predictions by 3 classifiers (`y_model_1`, `y_model_2`, and `y_model_3`): y_model_1 = np.array([1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]) y_model_2 = np.array([1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]) y_model_3 = np.array([1, 1, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1]) Assuming a significance level \\alpha=0.05 , we can conduct Cochran's Q test as follows, to test the null hypothesis there is no difference between the classification accuracies, p_i: H_0 = p_1 = p_2 = \\cdots = p_L : f, p_value = ftest(y_true, y_model_1, y_model_2, y_model_3) print('F: %.3f' % f) print('p-value: %.3f' % p_value) F: 3.873 p-value: 0.022 Since the p-value is smaller than \\alpha , we can reject the null hypothesis and conclude that there is a difference between the classification accuracies. As mentioned in the introduction earlier, we could now perform multiple post hoc pair-wise tests -- for example, McNemar tests with a Bonferroni correction -- to determine which pairs have different population proportions.","title":"Example 1 - F-test"},{"location":"user_guide/evaluate/ftest/#api","text":"ftest(y_target, y_model_predictions)* F-Test test to compare 2 or more models. Parameters y_target : array-like, shape=[n_samples] True class labels as 1D NumPy array. *y_model_predictions : array-likes, shape=[n_samples] Variable number of 2 or more arrays that contain the predicted class labels from models as 1D NumPy array. Returns f, p : float or None, float Returns the F-value and the p-value Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/ftest/","title":"API"},{"location":"user_guide/evaluate/lift_score/","text":"lift_score: Lift score for classification and association rule mining Scoring function to compute the LIFT metric, the ratio of correctly predicted positive examples and the actual positive examples in the test dataset. from mlxtend.evaluate import lift_score Overview In the context of classification, lift [1] compares model predictions to randomly generated predictions. Lift is often used in marketing research combined with gain and lift charts as a visual aid [2]. For example, assuming a 10% customer response as a baseline, a lift value of 3 would correspond to a 30% customer response when using the predictive model. Note that lift has the range \\lbrack 0, \\infty \\rbrack . There are many strategies to compute lift , and below, we will illustrate the computation of the lift score using a classic confusion matrix. For instance, let's assume the following prediction and target labels, where \"1\" is the positive class: \\text{true labels}: [0, 0, 1, 0, 0, 1, 1, 1, 1, 1] \\text{prediction}: [1, 0, 1, 0, 0, 0, 0, 1, 0, 0] Then, our confusion matrix would look as follows: Based on the confusion matrix above, with \"1\" as positive label, we compute lift as follows: \\text{lift} = \\frac{(TP/(TP+FP)}{(TP+FN)/(TP+TN+FP+FN)} Plugging in the actual values from the example above, we arrive at the following lift value: \\frac{2/(2+1)}{(2+4)/(2+3+1+4)} = 1.1111111111111112 An alternative way to computing lift is by using the support metric [3]: \\text{lift} = \\frac{\\text{support}(\\text{true labels} \\cap \\text{prediction})}{\\text{support}(\\text{true labels}) \\times \\text{support}(\\text{prediction})}, Support is x / N , where x is the number of incidences of an observation and N is the total number of samples in the datset. \\text{true labels} \\cap \\text{prediction} are the true positives, true labels are true positives plus false negatives, and prediction are true positives plus false positives. Plugging the values from our example into the equation above, we arrive at: \\frac{2/10}{(6/10 \\times 3/10)} = 1.1111111111111112 References [1] S. Brin, R. Motwani, J. D. Ullman, and S. Tsur. Dynamic itemset counting and implication rules for market basket data . In Proc. of the ACM SIGMOD Int'l Conf. on Management of Data (ACM SIGMOD '97), pages 265-276, 1997. [2] https://www3.nd.edu/~busiforc/Lift_chart.html [3] https://en.wikipedia.org/wiki/Association_rule_learning#Support Example 1 - Computing Lift This examples demonstrates the basic use of the lift_score function using the example from the Overview section. import numpy as np from mlxtend.evaluate import lift_score y_target = np.array([0, 0, 1, 0, 0, 1, 1, 1, 1, 1]) y_predicted = np.array([1, 0, 1, 0, 0, 0, 0, 1, 0, 0]) lift_score(y_target, y_predicted) 1.1111111111111112 Example 2 - Using lift_score in GridSearch The lift_score function can also be used with scikit-learn objects, such as GridSearch : from sklearn.datasets import load_iris from sklearn.model_selection import train_test_split from sklearn.model_selection import GridSearchCV from sklearn.svm import SVC from sklearn.metrics import make_scorer # make custom scorer lift_scorer = make_scorer(lift_score) iris = load_iris() X, y = iris.data, iris.target X_train, X_test, y_train, y_test = train_test_split( X, y, test_size=0.2, stratify=y, random_state=123) hyperparameters = [{'kernel': ['rbf'], 'gamma': [1e-3, 1e-4], 'C': [1, 10, 100, 1000]}, {'kernel': ['linear'], 'C': [1, 10, 100, 1000]}] clf = GridSearchCV(SVC(), hyperparameters, cv=10, scoring=lift_scorer) clf.fit(X_train, y_train) print(clf.best_score_) print(clf.best_params_) 3.0 {'gamma': 0.001, 'kernel': 'rbf', 'C': 1000} API lift_score(y_target, y_predicted, binary=True, positive_label=1) Lift measures the degree to which the predictions of a classification model are better than randomly-generated predictions. The in terms of True Positives (TP), True Negatives (TN), False Positives (FP), and False Negatives (FN), the lift score is computed as: [ TP/(TP+FN) ] / [ (TP+FP) / (TP+TN+FP+FN) ] Parameters y_target : array-like, shape=[n_samples] True class labels. y_predicted : array-like, shape=[n_samples] Predicted class labels. binary : bool (default: True) Maps a multi-class problem onto a binary, where the positive class is 1 and all other classes are 0. positive_label : int (default: 0) Class label of the positive class. Returns score : float Lift score in the range [0, \\infty ] Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/lift_score/","title":"lift_score: Lift score for classification and association rule mining"},{"location":"user_guide/evaluate/lift_score/#lift_score-lift-score-for-classification-and-association-rule-mining","text":"Scoring function to compute the LIFT metric, the ratio of correctly predicted positive examples and the actual positive examples in the test dataset. from mlxtend.evaluate import lift_score","title":"lift_score: Lift score for classification and association rule mining"},{"location":"user_guide/evaluate/lift_score/#overview","text":"In the context of classification, lift [1] compares model predictions to randomly generated predictions. Lift is often used in marketing research combined with gain and lift charts as a visual aid [2]. For example, assuming a 10% customer response as a baseline, a lift value of 3 would correspond to a 30% customer response when using the predictive model. Note that lift has the range \\lbrack 0, \\infty \\rbrack . There are many strategies to compute lift , and below, we will illustrate the computation of the lift score using a classic confusion matrix. For instance, let's assume the following prediction and target labels, where \"1\" is the positive class: \\text{true labels}: [0, 0, 1, 0, 0, 1, 1, 1, 1, 1] \\text{prediction}: [1, 0, 1, 0, 0, 0, 0, 1, 0, 0] Then, our confusion matrix would look as follows: Based on the confusion matrix above, with \"1\" as positive label, we compute lift as follows: \\text{lift} = \\frac{(TP/(TP+FP)}{(TP+FN)/(TP+TN+FP+FN)} Plugging in the actual values from the example above, we arrive at the following lift value: \\frac{2/(2+1)}{(2+4)/(2+3+1+4)} = 1.1111111111111112 An alternative way to computing lift is by using the support metric [3]: \\text{lift} = \\frac{\\text{support}(\\text{true labels} \\cap \\text{prediction})}{\\text{support}(\\text{true labels}) \\times \\text{support}(\\text{prediction})}, Support is x / N , where x is the number of incidences of an observation and N is the total number of samples in the datset. \\text{true labels} \\cap \\text{prediction} are the true positives, true labels are true positives plus false negatives, and prediction are true positives plus false positives. Plugging the values from our example into the equation above, we arrive at: \\frac{2/10}{(6/10 \\times 3/10)} = 1.1111111111111112","title":"Overview"},{"location":"user_guide/evaluate/lift_score/#references","text":"[1] S. Brin, R. Motwani, J. D. Ullman, and S. Tsur. Dynamic itemset counting and implication rules for market basket data . In Proc. of the ACM SIGMOD Int'l Conf. on Management of Data (ACM SIGMOD '97), pages 265-276, 1997. [2] https://www3.nd.edu/~busiforc/Lift_chart.html [3] https://en.wikipedia.org/wiki/Association_rule_learning#Support","title":"References"},{"location":"user_guide/evaluate/lift_score/#example-1-computing-lift","text":"This examples demonstrates the basic use of the lift_score function using the example from the Overview section. import numpy as np from mlxtend.evaluate import lift_score y_target = np.array([0, 0, 1, 0, 0, 1, 1, 1, 1, 1]) y_predicted = np.array([1, 0, 1, 0, 0, 0, 0, 1, 0, 0]) lift_score(y_target, y_predicted) 1.1111111111111112","title":"Example 1 - Computing Lift"},{"location":"user_guide/evaluate/lift_score/#example-2-using-lift_score-in-gridsearch","text":"The lift_score function can also be used with scikit-learn objects, such as GridSearch : from sklearn.datasets import load_iris from sklearn.model_selection import train_test_split from sklearn.model_selection import GridSearchCV from sklearn.svm import SVC from sklearn.metrics import make_scorer # make custom scorer lift_scorer = make_scorer(lift_score) iris = load_iris() X, y = iris.data, iris.target X_train, X_test, y_train, y_test = train_test_split( X, y, test_size=0.2, stratify=y, random_state=123) hyperparameters = [{'kernel': ['rbf'], 'gamma': [1e-3, 1e-4], 'C': [1, 10, 100, 1000]}, {'kernel': ['linear'], 'C': [1, 10, 100, 1000]}] clf = GridSearchCV(SVC(), hyperparameters, cv=10, scoring=lift_scorer) clf.fit(X_train, y_train) print(clf.best_score_) print(clf.best_params_) 3.0 {'gamma': 0.001, 'kernel': 'rbf', 'C': 1000}","title":"Example 2 - Using lift_score in GridSearch"},{"location":"user_guide/evaluate/lift_score/#api","text":"lift_score(y_target, y_predicted, binary=True, positive_label=1) Lift measures the degree to which the predictions of a classification model are better than randomly-generated predictions. The in terms of True Positives (TP), True Negatives (TN), False Positives (FP), and False Negatives (FN), the lift score is computed as: [ TP/(TP+FN) ] / [ (TP+FP) / (TP+TN+FP+FN) ] Parameters y_target : array-like, shape=[n_samples] True class labels. y_predicted : array-like, shape=[n_samples] Predicted class labels. binary : bool (default: True) Maps a multi-class problem onto a binary, where the positive class is 1 and all other classes are 0. positive_label : int (default: 0) Class label of the positive class. Returns score : float Lift score in the range [0, \\infty ] Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/lift_score/","title":"API"},{"location":"user_guide/evaluate/mcnemar/","text":"mcnemar: McNemar's test for classifier comparisons McNemar's test for paired nominal data from mlxtend.evaluate import mcnemar Overview McNemar's Test [1] (sometimes also called \"within-subjects chi-squared test\") is a statistical test for paired nominal data. In context of machine learning (or statistical) models, we can use McNemar's Test to compare the predictive accuracy of two models. McNemar's test is based on a 2 times 2 contingency table of the two model's predictions. McNemar's Test Statistic In McNemar's Test, we formulate the null hypothesis that the probabilities p(b) and p(c) are the same, or in simplified terms: None of the two models performs better than the other. Thus, the alternative hypothesis is that the performances of the two models are not equal. The McNemar test statistic (\"chi-squared\") can be computed as follows: \\chi^2 = \\frac{(b - c)^2}{(b + c)}, If the sum of cell c and b is sufficiently large, the \\chi^2 value follows a chi-squared distribution with one degree of freedom. After setting a significance threshold, e.g,. \\alpha=0.05 we can compute the p-value -- assuming that the null hypothesis is true, the p-value is the probability of observing this empirical (or a larger) chi-squared value. If the p-value is lower than our chosen significance level, we can reject the null hypothesis that the two model's performances are equal. Continuity Correction Approximately 1 year after Quinn McNemar published the McNemar Test [1], Edwards [2] proposed a continuity corrected version, which is the more commonly used variant today: \\chi^2 = \\frac{( \\mid b - c \\mid - 1)^2}{(b + c)}. Exact p-values As mentioned earlier, an exact binomial test is recommended for small sample sizes ( b + c < 25 [3]), since the chi-squared value may not be well-approximated by the chi-squared distribution. The exact p-value can be computed as follows: p = 2 \\sum^{n}_{i=b} \\binom{n}{i} 0.5^i (1 - 0.5)^{n-i}, where n = b + c , and the factor 2 is used to compute the two-sided p-value. Example For instance, given that 2 models have a accuracy of with a 99.7% and 99.6% a 2x2 contingency table can provide further insights for model selection. In both subfigure A and B, the predictive accuracies of the two models are as follows: model 1 accuracy: 9,960 / 10,000 = 99.6% model 2 accuracy: 9,970 / 10,000 = 99.7% Now, in subfigure A, we can see that model 2 got 11 predictions right that model 1 got wrong. Vice versa, model 2 got 1 prediction right that model 2 got wrong. Thus, based on this 11:1 ratio, we may conclude that model 2 performs substantially better than model 1. However, in subfigure B, the ratio is 25:15, which is less conclusive about which model is the better one to choose. In the following coding examples, we will use these 2 scenarios A and B to illustrate McNemar's test. References [1] McNemar, Quinn, 1947. \" Note on the sampling error of the difference between correlated proportions or percentages \". Psychometrika. 12 (2): 153\u2013157. [2] Edwards AL: Note on the \u201ccorrection for continuity\u201d in testing the significance of the difference between correlated proportions. Psychometrika. 1948, 13 (3): 185-187. 10.1007/BF02289261. [3] https://en.wikipedia.org/wiki/McNemar%27s_test Example 1 - Creating 2x2 contingency tables The mcnemar funtion expects a 2x2 contingency table as a NumPy array that is formatted as follows: Such a contingency matrix can be created by using the mcnemar_table function from mlxtend.evaluate . For example: import numpy as np from mlxtend.evaluate import mcnemar_table # The correct target (class) labels y_target = np.array([0, 0, 0, 0, 0, 1, 1, 1, 1, 1]) # Class labels predicted by model 1 y_model1 = np.array([0, 1, 0, 0, 0, 1, 1, 0, 0, 0]) # Class labels predicted by model 2 y_model2 = np.array([0, 0, 1, 1, 0, 1, 1, 0, 0, 0]) tb = mcnemar_table(y_target=y_target, y_model1=y_model1, y_model2=y_model2) print(tb) [[4 1] [2 3]] Example 2 - McNemar's Test for Scenario B No, let us continue with the example mentioned in the overview section and assume that we already computed the 2x2 contingency table: import numpy as np tb_b = np.array([[9945, 25], [15, 15]]) To test the null hypothesis that the predictive performance of two models are equal (using a significance level of \\alpha=0.05 ), we can conduct a corrected McNemar Test for computing the chi-squared and p-value as follows: from mlxtend.evaluate import mcnemar chi2, p = mcnemar(ary=tb_b, corrected=True) print('chi-squared:', chi2) print('p-value:', p) chi-squared: 2.025 p-value: 0.154728923485 Since the p-value is larger than our assumed significance threshold ( \\alpha=0.05 ), we cannot reject our null hypothesis and assume that there is no significant difference between the two predictive models. Example 3 - McNemar's Test for Scenario A In contrast to scenario B (Example 2), the sample size in scenario A is relatively small (b + c = 11 + 1 = 12) and smaller than the recommended 25 [3] to approximate the computed chi-square value by the chi-square distribution well. In this case, we need to compute the exact p-value from the binomial distribution: from mlxtend.evaluate import mcnemar import numpy as np tb_a = np.array([[9959, 11], [1, 29]]) chi2, p = mcnemar(ary=tb_a, exact=True) print('chi-squared:', chi2) print('p-value:', p) chi-squared: None p-value: 0.005859375 Assuming that we conducted this test also with a significance level of \\alpha=0.05 , we can reject the null-hypothesis that both models perform equally well on this dataset, since the p-value ( p \\approx 0.006 ) is smaller than \\alpha . API mcnemar(ary, corrected=True, exact=False) McNemar test for paired nominal data Parameters ary : array-like, shape=[2, 2] 2 x 2 contingency table (as returned by evaluate.mcnemar_table), where a: ary[0, 0]: # of samples that both models predicted correctly b: ary[0, 1]: # of samples that model 1 got right and model 2 got wrong c: ary[1, 0]: # of samples that model 2 got right and model 1 got wrong d: aryCell [1, 1]: # of samples that both models predicted incorrectly corrected : array-like, shape=[n_samples] (default: True) Uses Edward's continuity correction for chi-squared if True exact : bool, (default: False) If True , uses an exact binomial test comparing b to a binomial distribution with n = b + c and p = 0.5. It is highly recommended to use exact=True for sample sizes < 25 since chi-squared is not well-approximated by the chi-squared distribution! Returns chi2, p : float or None, float Returns the chi-squared value and the p-value; if exact=True (default: False ), chi2 is None Examples For usage examples, please see [https://rasbt.github.io/mlxtend/user_guide/evaluate/mcnemar/](https://rasbt.github.io/mlxtend/user_guide/evaluate/mcnemar/) ython","title":"mcnemar: McNemar's test for classifier comparisons"},{"location":"user_guide/evaluate/mcnemar/#mcnemar-mcnemars-test-for-classifier-comparisons","text":"McNemar's test for paired nominal data from mlxtend.evaluate import mcnemar","title":"mcnemar: McNemar's test for classifier comparisons"},{"location":"user_guide/evaluate/mcnemar/#overview","text":"McNemar's Test [1] (sometimes also called \"within-subjects chi-squared test\") is a statistical test for paired nominal data. In context of machine learning (or statistical) models, we can use McNemar's Test to compare the predictive accuracy of two models. McNemar's test is based on a 2 times 2 contingency table of the two model's predictions.","title":"Overview"},{"location":"user_guide/evaluate/mcnemar/#mcnemars-test-statistic","text":"In McNemar's Test, we formulate the null hypothesis that the probabilities p(b) and p(c) are the same, or in simplified terms: None of the two models performs better than the other. Thus, the alternative hypothesis is that the performances of the two models are not equal. The McNemar test statistic (\"chi-squared\") can be computed as follows: \\chi^2 = \\frac{(b - c)^2}{(b + c)}, If the sum of cell c and b is sufficiently large, the \\chi^2 value follows a chi-squared distribution with one degree of freedom. After setting a significance threshold, e.g,. \\alpha=0.05 we can compute the p-value -- assuming that the null hypothesis is true, the p-value is the probability of observing this empirical (or a larger) chi-squared value. If the p-value is lower than our chosen significance level, we can reject the null hypothesis that the two model's performances are equal.","title":"McNemar's Test Statistic"},{"location":"user_guide/evaluate/mcnemar/#continuity-correction","text":"Approximately 1 year after Quinn McNemar published the McNemar Test [1], Edwards [2] proposed a continuity corrected version, which is the more commonly used variant today: \\chi^2 = \\frac{( \\mid b - c \\mid - 1)^2}{(b + c)}.","title":"Continuity Correction"},{"location":"user_guide/evaluate/mcnemar/#exact-p-values","text":"As mentioned earlier, an exact binomial test is recommended for small sample sizes ( b + c < 25 [3]), since the chi-squared value may not be well-approximated by the chi-squared distribution. The exact p-value can be computed as follows: p = 2 \\sum^{n}_{i=b} \\binom{n}{i} 0.5^i (1 - 0.5)^{n-i}, where n = b + c , and the factor 2 is used to compute the two-sided p-value.","title":"Exact p-values"},{"location":"user_guide/evaluate/mcnemar/#example","text":"For instance, given that 2 models have a accuracy of with a 99.7% and 99.6% a 2x2 contingency table can provide further insights for model selection. In both subfigure A and B, the predictive accuracies of the two models are as follows: model 1 accuracy: 9,960 / 10,000 = 99.6% model 2 accuracy: 9,970 / 10,000 = 99.7% Now, in subfigure A, we can see that model 2 got 11 predictions right that model 1 got wrong. Vice versa, model 2 got 1 prediction right that model 2 got wrong. Thus, based on this 11:1 ratio, we may conclude that model 2 performs substantially better than model 1. However, in subfigure B, the ratio is 25:15, which is less conclusive about which model is the better one to choose. In the following coding examples, we will use these 2 scenarios A and B to illustrate McNemar's test.","title":"Example"},{"location":"user_guide/evaluate/mcnemar/#references","text":"[1] McNemar, Quinn, 1947. \" Note on the sampling error of the difference between correlated proportions or percentages \". Psychometrika. 12 (2): 153\u2013157. [2] Edwards AL: Note on the \u201ccorrection for continuity\u201d in testing the significance of the difference between correlated proportions. Psychometrika. 1948, 13 (3): 185-187. 10.1007/BF02289261. [3] https://en.wikipedia.org/wiki/McNemar%27s_test","title":"References"},{"location":"user_guide/evaluate/mcnemar/#example-1-creating-2x2-contingency-tables","text":"The mcnemar funtion expects a 2x2 contingency table as a NumPy array that is formatted as follows: Such a contingency matrix can be created by using the mcnemar_table function from mlxtend.evaluate . For example: import numpy as np from mlxtend.evaluate import mcnemar_table # The correct target (class) labels y_target = np.array([0, 0, 0, 0, 0, 1, 1, 1, 1, 1]) # Class labels predicted by model 1 y_model1 = np.array([0, 1, 0, 0, 0, 1, 1, 0, 0, 0]) # Class labels predicted by model 2 y_model2 = np.array([0, 0, 1, 1, 0, 1, 1, 0, 0, 0]) tb = mcnemar_table(y_target=y_target, y_model1=y_model1, y_model2=y_model2) print(tb) [[4 1] [2 3]]","title":"Example 1 - Creating 2x2 contingency tables"},{"location":"user_guide/evaluate/mcnemar/#example-2-mcnemars-test-for-scenario-b","text":"No, let us continue with the example mentioned in the overview section and assume that we already computed the 2x2 contingency table: import numpy as np tb_b = np.array([[9945, 25], [15, 15]]) To test the null hypothesis that the predictive performance of two models are equal (using a significance level of \\alpha=0.05 ), we can conduct a corrected McNemar Test for computing the chi-squared and p-value as follows: from mlxtend.evaluate import mcnemar chi2, p = mcnemar(ary=tb_b, corrected=True) print('chi-squared:', chi2) print('p-value:', p) chi-squared: 2.025 p-value: 0.154728923485 Since the p-value is larger than our assumed significance threshold ( \\alpha=0.05 ), we cannot reject our null hypothesis and assume that there is no significant difference between the two predictive models.","title":"Example 2 - McNemar's Test for Scenario B"},{"location":"user_guide/evaluate/mcnemar/#example-3-mcnemars-test-for-scenario-a","text":"In contrast to scenario B (Example 2), the sample size in scenario A is relatively small (b + c = 11 + 1 = 12) and smaller than the recommended 25 [3] to approximate the computed chi-square value by the chi-square distribution well. In this case, we need to compute the exact p-value from the binomial distribution: from mlxtend.evaluate import mcnemar import numpy as np tb_a = np.array([[9959, 11], [1, 29]]) chi2, p = mcnemar(ary=tb_a, exact=True) print('chi-squared:', chi2) print('p-value:', p) chi-squared: None p-value: 0.005859375 Assuming that we conducted this test also with a significance level of \\alpha=0.05 , we can reject the null-hypothesis that both models perform equally well on this dataset, since the p-value ( p \\approx 0.006 ) is smaller than \\alpha .","title":"Example 3 - McNemar's Test for Scenario A"},{"location":"user_guide/evaluate/mcnemar/#api","text":"mcnemar(ary, corrected=True, exact=False) McNemar test for paired nominal data Parameters ary : array-like, shape=[2, 2] 2 x 2 contingency table (as returned by evaluate.mcnemar_table), where a: ary[0, 0]: # of samples that both models predicted correctly b: ary[0, 1]: # of samples that model 1 got right and model 2 got wrong c: ary[1, 0]: # of samples that model 2 got right and model 1 got wrong d: aryCell [1, 1]: # of samples that both models predicted incorrectly corrected : array-like, shape=[n_samples] (default: True) Uses Edward's continuity correction for chi-squared if True exact : bool, (default: False) If True , uses an exact binomial test comparing b to a binomial distribution with n = b + c and p = 0.5. It is highly recommended to use exact=True for sample sizes < 25 since chi-squared is not well-approximated by the chi-squared distribution! Returns chi2, p : float or None, float Returns the chi-squared value and the p-value; if exact=True (default: False ), chi2 is None Examples For usage examples, please see [https://rasbt.github.io/mlxtend/user_guide/evaluate/mcnemar/](https://rasbt.github.io/mlxtend/user_guide/evaluate/mcnemar/) ython","title":"API"},{"location":"user_guide/evaluate/mcnemar_table/","text":"mcnemar_table: Contingency table for McNemar's test Function to compute a 2x2 contingency table for McNemar's Test from mlxtend.evaluate import mcnemar_table Overview contingency Table for McNemar's Test A 2x2 contingency table as being used in a McNemar's Test ( mlxtend.evaluate.mcnemar ) is a useful aid for comparing two different models. In contrast to a typical confusion matrix, this table compares two models to each other rather than showing the false positives, true positives, false negatives, and true negatives of a single model's predictions: For instance, given that 2 models have a accuracy of with a 99.7% and 99.6% a 2x2 contingency table can provide further insights for model selection. In both subfigure A and B, the predictive accuracies of the two models are as follows: model 1 accuracy: 9,960 / 10,000 = 99.6% model 2 accuracy: 9,970 / 10,000 = 99.7% Now, in subfigure A, we can see that model 2 got 11 predictions right that model 1 got wrong. Vice versa, model 2 got 1 prediction right that model 2 got wrong. Thus, based on this 11:1 ratio, we may conclude that model 2 performs substantially better than model 1. However, in subfigure B, the ratio is 25:15, which is less conclusive about which model is the better one to choose. References McNemar, Quinn, 1947. \" Note on the sampling error of the difference between correlated proportions or percentages \". Psychometrika. 12 (2): 153\u2013157. Edwards AL: Note on the \u201ccorrection for continuity\u201d in testing the significance of the difference between correlated proportions. Psychometrika. 1948, 13 (3): 185-187. 10.1007/BF02289261. https://en.wikipedia.org/wiki/McNemar%27s_test Example 2 - 2x2 contingency Table import numpy as np from mlxtend.evaluate import mcnemar_table y_true = np.array([0, 0, 0, 0, 0, 1, 1, 1, 1, 1]) y_mod1 = np.array([0, 1, 0, 0, 0, 1, 1, 0, 0, 0]) y_mod2 = np.array([0, 0, 1, 1, 0, 1, 1, 0, 0, 0]) tb = mcnemar_table(y_target=y_true, y_model1=y_mod1, y_model2=y_mod2) tb array([[4, 1], [2, 3]]) To visualize (and better interpret) the contingency table via matplotlib, we can use the checkerboard_plot function: from mlxtend.plotting import checkerboard_plot import matplotlib.pyplot as plt brd = checkerboard_plot(tb, figsize=(3, 3), fmt='%d', col_labels=['model 2 wrong', 'model 2 right'], row_labels=['model 1 wrong', 'model 1 right']) plt.show() API mcnemar_table(y_target, y_model1, y_model2) Compute a 2x2 contingency table for McNemar's test. Parameters y_target : array-like, shape=[n_samples] True class labels as 1D NumPy array. y_model1 : array-like, shape=[n_samples] Predicted class labels from model as 1D NumPy array. y_model2 : array-like, shape=[n_samples] Predicted class labels from model 2 as 1D NumPy array. Returns tb : array-like, shape=[2, 2] 2x2 contingency table with the following contents: a: tb[0, 0]: # of samples that both models predicted correctly b: tb[0, 1]: # of samples that model 1 got right and model 2 got wrong c: tb[1, 0]: # of samples that model 2 got right and model 1 got wrong d: tb[1, 1]: # of samples that both models predicted incorrectly Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/mcnemar_table/","title":"mcnemar_table: Contingency table for McNemar's test"},{"location":"user_guide/evaluate/mcnemar_table/#mcnemar_table-contingency-table-for-mcnemars-test","text":"Function to compute a 2x2 contingency table for McNemar's Test from mlxtend.evaluate import mcnemar_table","title":"mcnemar_table: Contingency table for McNemar's test"},{"location":"user_guide/evaluate/mcnemar_table/#overview","text":"","title":"Overview"},{"location":"user_guide/evaluate/mcnemar_table/#contingency-table-for-mcnemars-test","text":"A 2x2 contingency table as being used in a McNemar's Test ( mlxtend.evaluate.mcnemar ) is a useful aid for comparing two different models. In contrast to a typical confusion matrix, this table compares two models to each other rather than showing the false positives, true positives, false negatives, and true negatives of a single model's predictions: For instance, given that 2 models have a accuracy of with a 99.7% and 99.6% a 2x2 contingency table can provide further insights for model selection. In both subfigure A and B, the predictive accuracies of the two models are as follows: model 1 accuracy: 9,960 / 10,000 = 99.6% model 2 accuracy: 9,970 / 10,000 = 99.7% Now, in subfigure A, we can see that model 2 got 11 predictions right that model 1 got wrong. Vice versa, model 2 got 1 prediction right that model 2 got wrong. Thus, based on this 11:1 ratio, we may conclude that model 2 performs substantially better than model 1. However, in subfigure B, the ratio is 25:15, which is less conclusive about which model is the better one to choose.","title":"contingency Table for McNemar's Test"},{"location":"user_guide/evaluate/mcnemar_table/#references","text":"McNemar, Quinn, 1947. \" Note on the sampling error of the difference between correlated proportions or percentages \". Psychometrika. 12 (2): 153\u2013157. Edwards AL: Note on the \u201ccorrection for continuity\u201d in testing the significance of the difference between correlated proportions. Psychometrika. 1948, 13 (3): 185-187. 10.1007/BF02289261. https://en.wikipedia.org/wiki/McNemar%27s_test","title":"References"},{"location":"user_guide/evaluate/mcnemar_table/#example-2-2x2-contingency-table","text":"import numpy as np from mlxtend.evaluate import mcnemar_table y_true = np.array([0, 0, 0, 0, 0, 1, 1, 1, 1, 1]) y_mod1 = np.array([0, 1, 0, 0, 0, 1, 1, 0, 0, 0]) y_mod2 = np.array([0, 0, 1, 1, 0, 1, 1, 0, 0, 0]) tb = mcnemar_table(y_target=y_true, y_model1=y_mod1, y_model2=y_mod2) tb array([[4, 1], [2, 3]]) To visualize (and better interpret) the contingency table via matplotlib, we can use the checkerboard_plot function: from mlxtend.plotting import checkerboard_plot import matplotlib.pyplot as plt brd = checkerboard_plot(tb, figsize=(3, 3), fmt='%d', col_labels=['model 2 wrong', 'model 2 right'], row_labels=['model 1 wrong', 'model 1 right']) plt.show()","title":"Example 2 - 2x2 contingency Table"},{"location":"user_guide/evaluate/mcnemar_table/#api","text":"mcnemar_table(y_target, y_model1, y_model2) Compute a 2x2 contingency table for McNemar's test. Parameters y_target : array-like, shape=[n_samples] True class labels as 1D NumPy array. y_model1 : array-like, shape=[n_samples] Predicted class labels from model as 1D NumPy array. y_model2 : array-like, shape=[n_samples] Predicted class labels from model 2 as 1D NumPy array. Returns tb : array-like, shape=[2, 2] 2x2 contingency table with the following contents: a: tb[0, 0]: # of samples that both models predicted correctly b: tb[0, 1]: # of samples that model 1 got right and model 2 got wrong c: tb[1, 0]: # of samples that model 2 got right and model 1 got wrong d: tb[1, 1]: # of samples that both models predicted incorrectly Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/mcnemar_table/","title":"API"},{"location":"user_guide/evaluate/mcnemar_tables/","text":"mcnemar_tables: contingency tables for McNemar's test and Cochran's Q test Function to compute a 2x2 contingency tables for McNemar's Test and Cochran's Q Test from mlxtend.evaluate import mcnemar_tables Overview contingency Tables A 2x2 contingency table as being used in a McNemar's Test ( mlxtend.evaluate.mcnemar ) is a useful aid for comparing two different models. In contrast to a typical confusion matrix, this table compares two models to each other rather than showing the false positives, true positives, false negatives, and true negatives of a single model's predictions: For instance, given that 2 models have a accuracy of with a 99.7% and 99.6% a 2x2 contingency table can provide further insights for model selection. In both subfigure A and B, the predictive accuracies of the two models are as follows: model 1 accuracy: 9,960 / 10,000 = 99.6% model 2 accuracy: 9,970 / 10,000 = 99.7% Now, in subfigure A, we can see that model 2 got 11 predictions right that model 1 got wrong. Vice versa, model 2 got 1 prediction right that model 2 got wrong. Thus, based on this 11:1 ratio, we may conclude that model 2 performs substantially better than model 1. However, in subfigure B, the ratio is 25:15, which is less conclusive about which model is the better one to choose. References McNemar, Quinn, 1947. \" Note on the sampling error of the difference between correlated proportions or percentages \". Psychometrika. 12 (2): 153\u2013157. Edwards AL: Note on the \u201ccorrection for continuity\u201d in testing the significance of the difference between correlated proportions. Psychometrika. 1948, 13 (3): 185-187. 10.1007/BF02289261. https://en.wikipedia.org/wiki/McNemar%27s_test Example 1 - Single 2x2 contingency Table import numpy as np from mlxtend.evaluate import mcnemar_tables y_true = np.array([0, 0, 0, 0, 0, 1, 1, 1, 1, 1]) y_mod0 = np.array([0, 1, 0, 0, 0, 1, 1, 0, 0, 0]) y_mod1 = np.array([0, 0, 1, 1, 0, 1, 1, 0, 0, 0]) tb = mcnemar_tables(y_true, y_mod0, y_mod1) tb {'model_0 vs model_1': array([[ 4., 1.], [ 2., 3.]])} To visualize (and better interpret) the contingency table via matplotlib, we can use the checkerboard_plot function: from mlxtend.plotting import checkerboard_plot import matplotlib.pyplot as plt brd = checkerboard_plot(tb['model_0 vs model_1'], figsize=(3, 3), fmt='%d', col_labels=['model 2 wrong', 'model 2 right'], row_labels=['model 1 wrong', 'model 1 right']) plt.show() Example 2 - Multiple 2x2 contingency Tables If more than two models are provided as input to the mcnemar_tables function, a 2x2 contingency table will be created for each pair of models: import numpy as np from mlxtend.evaluate import mcnemar_tables y_true = np.array([0, 0, 0, 0, 0, 1, 1, 1, 1, 1]) y_mod0 = np.array([0, 1, 0, 0, 0, 1, 1, 0, 0, 0]) y_mod1 = np.array([0, 0, 1, 1, 0, 1, 1, 0, 0, 0]) y_mod2 = np.array([0, 0, 1, 1, 0, 1, 1, 0, 1, 0]) tb = mcnemar_tables(y_true, y_mod0, y_mod1, y_mod2) for key, value in tb.items(): print(key, '\\n', value, '\\n') model_0 vs model_1 [[ 4. 1.] [ 2. 3.]] model_0 vs model_2 [[ 4. 2.] [ 2. 2.]] model_1 vs model_2 [[ 5. 1.] [ 0. 4.]] API mcnemar_tables(y_target, y_model_predictions)* Compute multiple 2x2 contigency tables for McNemar's test or Cochran's Q test. Parameters y_target : array-like, shape=[n_samples] True class labels as 1D NumPy array. y_model_predictions : array-like, shape=[n_samples] Predicted class labels for a model. Returns tables : dict Dictionary of NumPy arrays with shape=[2, 2]. Each dictionary key names the two models to be compared based on the order the models were passed as *y_model_predictions . The number of dictionary entries is equal to the number of pairwise combinations between the m models, i.e., \"m choose 2.\" For example the following target array (containing the true labels) and 3 models y_true = np.array([0, 0, 0, 0, 0, 1, 1, 1, 1, 1]) y_mod0 = np.array([0, 1, 0, 0, 0, 1, 1, 0, 0, 0]) y_mod1 = np.array([0, 0, 1, 1, 0, 1, 1, 0, 0, 0]) y_mod2 = np.array([0, 1, 1, 1, 0, 1, 0, 0, 0, 0]) would result in the following dictionary: {'model_0 vs model_1': array([[ 4., 1.], [ 2., 3.]]), 'model_0 vs model_2': array([[ 3., 0.], [ 3., 4.]]), 'model_1 vs model_2': array([[ 3., 0.], [ 2., 5.]])} Each array is structured in the following way: tb[0, 0]: # of samples that both models predicted correctly tb[0, 1]: # of samples that model a got right and model b got wrong tb[1, 0]: # of samples that model b got right and model a got wrong tb[1, 1]: # of samples that both models predicted incorrectly Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/mcnemar_tables/ ython","title":"mcnemar_tables: contingency tables for McNemar's test and Cochran's Q test"},{"location":"user_guide/evaluate/mcnemar_tables/#mcnemar_tables-contingency-tables-for-mcnemars-test-and-cochrans-q-test","text":"Function to compute a 2x2 contingency tables for McNemar's Test and Cochran's Q Test from mlxtend.evaluate import mcnemar_tables","title":"mcnemar_tables: contingency tables for McNemar's test and Cochran's Q test"},{"location":"user_guide/evaluate/mcnemar_tables/#overview","text":"","title":"Overview"},{"location":"user_guide/evaluate/mcnemar_tables/#contingency-tables","text":"A 2x2 contingency table as being used in a McNemar's Test ( mlxtend.evaluate.mcnemar ) is a useful aid for comparing two different models. In contrast to a typical confusion matrix, this table compares two models to each other rather than showing the false positives, true positives, false negatives, and true negatives of a single model's predictions: For instance, given that 2 models have a accuracy of with a 99.7% and 99.6% a 2x2 contingency table can provide further insights for model selection. In both subfigure A and B, the predictive accuracies of the two models are as follows: model 1 accuracy: 9,960 / 10,000 = 99.6% model 2 accuracy: 9,970 / 10,000 = 99.7% Now, in subfigure A, we can see that model 2 got 11 predictions right that model 1 got wrong. Vice versa, model 2 got 1 prediction right that model 2 got wrong. Thus, based on this 11:1 ratio, we may conclude that model 2 performs substantially better than model 1. However, in subfigure B, the ratio is 25:15, which is less conclusive about which model is the better one to choose.","title":"contingency Tables"},{"location":"user_guide/evaluate/mcnemar_tables/#references","text":"McNemar, Quinn, 1947. \" Note on the sampling error of the difference between correlated proportions or percentages \". Psychometrika. 12 (2): 153\u2013157. Edwards AL: Note on the \u201ccorrection for continuity\u201d in testing the significance of the difference between correlated proportions. Psychometrika. 1948, 13 (3): 185-187. 10.1007/BF02289261. https://en.wikipedia.org/wiki/McNemar%27s_test","title":"References"},{"location":"user_guide/evaluate/mcnemar_tables/#example-1-single-2x2-contingency-table","text":"import numpy as np from mlxtend.evaluate import mcnemar_tables y_true = np.array([0, 0, 0, 0, 0, 1, 1, 1, 1, 1]) y_mod0 = np.array([0, 1, 0, 0, 0, 1, 1, 0, 0, 0]) y_mod1 = np.array([0, 0, 1, 1, 0, 1, 1, 0, 0, 0]) tb = mcnemar_tables(y_true, y_mod0, y_mod1) tb {'model_0 vs model_1': array([[ 4., 1.], [ 2., 3.]])} To visualize (and better interpret) the contingency table via matplotlib, we can use the checkerboard_plot function: from mlxtend.plotting import checkerboard_plot import matplotlib.pyplot as plt brd = checkerboard_plot(tb['model_0 vs model_1'], figsize=(3, 3), fmt='%d', col_labels=['model 2 wrong', 'model 2 right'], row_labels=['model 1 wrong', 'model 1 right']) plt.show()","title":"Example 1 - Single 2x2 contingency Table"},{"location":"user_guide/evaluate/mcnemar_tables/#example-2-multiple-2x2-contingency-tables","text":"If more than two models are provided as input to the mcnemar_tables function, a 2x2 contingency table will be created for each pair of models: import numpy as np from mlxtend.evaluate import mcnemar_tables y_true = np.array([0, 0, 0, 0, 0, 1, 1, 1, 1, 1]) y_mod0 = np.array([0, 1, 0, 0, 0, 1, 1, 0, 0, 0]) y_mod1 = np.array([0, 0, 1, 1, 0, 1, 1, 0, 0, 0]) y_mod2 = np.array([0, 0, 1, 1, 0, 1, 1, 0, 1, 0]) tb = mcnemar_tables(y_true, y_mod0, y_mod1, y_mod2) for key, value in tb.items(): print(key, '\\n', value, '\\n') model_0 vs model_1 [[ 4. 1.] [ 2. 3.]] model_0 vs model_2 [[ 4. 2.] [ 2. 2.]] model_1 vs model_2 [[ 5. 1.] [ 0. 4.]]","title":"Example 2 - Multiple 2x2 contingency Tables"},{"location":"user_guide/evaluate/mcnemar_tables/#api","text":"mcnemar_tables(y_target, y_model_predictions)* Compute multiple 2x2 contigency tables for McNemar's test or Cochran's Q test. Parameters y_target : array-like, shape=[n_samples] True class labels as 1D NumPy array. y_model_predictions : array-like, shape=[n_samples] Predicted class labels for a model. Returns tables : dict Dictionary of NumPy arrays with shape=[2, 2]. Each dictionary key names the two models to be compared based on the order the models were passed as *y_model_predictions . The number of dictionary entries is equal to the number of pairwise combinations between the m models, i.e., \"m choose 2.\" For example the following target array (containing the true labels) and 3 models y_true = np.array([0, 0, 0, 0, 0, 1, 1, 1, 1, 1]) y_mod0 = np.array([0, 1, 0, 0, 0, 1, 1, 0, 0, 0]) y_mod1 = np.array([0, 0, 1, 1, 0, 1, 1, 0, 0, 0]) y_mod2 = np.array([0, 1, 1, 1, 0, 1, 0, 0, 0, 0]) would result in the following dictionary: {'model_0 vs model_1': array([[ 4., 1.], [ 2., 3.]]), 'model_0 vs model_2': array([[ 3., 0.], [ 3., 4.]]), 'model_1 vs model_2': array([[ 3., 0.], [ 2., 5.]])} Each array is structured in the following way: tb[0, 0]: # of samples that both models predicted correctly tb[0, 1]: # of samples that model a got right and model b got wrong tb[1, 0]: # of samples that model b got right and model a got wrong tb[1, 1]: # of samples that both models predicted incorrectly Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/mcnemar_tables/ ython","title":"API"},{"location":"user_guide/evaluate/paired_ttest_5x2cv/","text":"paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons 5x2cv paired t test procedure to compare the performance of two models from mlxtend.evaluate import paired_ttest_5x2cv Overview The 5x2cv paired t test is a procedure for comparing the performance of two models (classifiers or regressors) that was proposed by Dietterich [1] to address shortcomings in other methods such as the resampled paired t test (see paired_ttest_resampled ) and the k-fold cross-validated paired t test (see paired_ttest_kfold_cv ). To explain how this method works, let's consider to estimator (e.g., classifiers) A and B. Further, we have a labeled dataset D . In the common hold-out method, we typically split the dataset into 2 parts: a training and a test set. In the 5x2cv paired t test, we repeat the splitting (50% training and 50% test data) 5 times. In each of the 5 iterations, we fit A and B to the training split and evaluate their performance ( p_A and p_B ) on the test split. Then, we rotate the training and test sets (the training set becomes the test set and vice versa) compute the performance again, which results in 2 performance difference measures: p^{(1)} = p^{(1)}_A - p^{(1)}_B and p^{(2)} = p^{(2)}_A - p^{(2)}_B. Then, we estimate the estimate mean and variance of the differences: \\overline{p} = \\frac{p^{(1)} + p^{(2)}}{2} and s^2 = (p^{(1)} - \\overline{p})^2 + (p^{(2)} - \\overline{p})^2. The variance of the difference is computed for the 5 iterations and then used to compute the t statistic as follows: t = \\frac{p_1^{(1)}}{\\sqrt{(1/5) \\sum_{i=1}^{5}s_i^2}}, where p_1^{(1)} is the p_1 from the very first iteration. The t statistic, assuming that it approximately follows as t distribution with 5 degrees of freedom, under the null hypothesis that the models A and B have equal performance. Using the t statistic, the p value can be computed and compared with a previously chosen significance level, e.g., \\alpha=0.05 . If the p value is smaller than \\alpha , we reject the null hypothesis and accept that there is a significant difference in the two models. References [1] Dietterich TG (1998) Approximate Statistical Tests for Comparing Supervised Classification Learning Algorithms. Neural Comput 10:1895\u20131923. Example 1 - 5x2cv paired t test Assume we want to compare two classification algorithms, logistic regression and a decision tree algorithm: from sklearn.linear_model import LogisticRegression from sklearn.tree import DecisionTreeClassifier from mlxtend.data import iris_data from sklearn.model_selection import train_test_split X, y = iris_data() clf1 = LogisticRegression(random_state=1) clf2 = DecisionTreeClassifier(random_state=1) X_train, X_test, y_train, y_test = \\ train_test_split(X, y, test_size=0.25, random_state=123) score1 = clf1.fit(X_train, y_train).score(X_test, y_test) score2 = clf2.fit(X_train, y_train).score(X_test, y_test) print('Logistic regression accuracy: %.2f%%' % (score1*100)) print('Decision tree accuracy: %.2f%%' % (score2*100)) Logistic regression accuracy: 97.37% Decision tree accuracy: 94.74% Note that these accuracy values are not used in the paired t test procedure as new test/train splits are generated during the resampling procedure, the values above are just serving the purpose of intuition. Now, let's assume a significance threshold of \\alpha=0.05 for rejecting the null hypothesis that both algorithms perform equally well on the dataset and conduct the 5x2cv t test: from mlxtend.evaluate import paired_ttest_5x2cv t, p = paired_ttest_5x2cv(estimator1=clf1, estimator2=clf2, X=X, y=y, random_seed=1) print('t statistic: %.3f' % t) print('p value: %.3f' % p) t statistic: -1.539 p value: 0.184 Since p > \\alpha , we cannot reject the null hypothesis and may conclude that the performance of the two algorithms is not significantly different. While it is generally not recommended to apply statistical tests multiple times without correction for multiple hypothesis testing, let us take a look at an example where the decision tree algorithm is limited to producing a very simple decision boundary that would result in a relatively bad performance: clf2 = DecisionTreeClassifier(random_state=1, max_depth=1) score2 = clf2.fit(X_train, y_train).score(X_test, y_test) print('Decision tree accuracy: %.2f%%' % (score2*100)) t, p = paired_ttest_5x2cv(estimator1=clf1, estimator2=clf2, X=X, y=y, random_seed=1) print('t statistic: %.3f' % t) print('p value: %.3f' % p) Decision tree accuracy: 63.16% t statistic: 5.386 p value: 0.003 Assuming that we conducted this test also with a significance level of \\alpha=0.05 , we can reject the null-hypothesis that both models perform equally well on this dataset, since the p-value ( p < 0.001 ) is smaller than \\alpha . API paired_ttest_5x2cv(estimator1, estimator2, X, y, scoring=None, random_seed=None) Implements the 5x2cv paired t test proposed by Dieterrich (1998) to compare the performance of two models. Parameters estimator1 : scikit-learn classifier or regressor estimator2 : scikit-learn classifier or regressor X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values. scoring : str, callable, or None (default: None) If None (default), uses 'accuracy' for sklearn classifiers and 'r2' for sklearn regressors. If str, uses a sklearn scoring metric string identifier, for example {accuracy, f1, precision, recall, roc_auc} for classifiers, {'mean_absolute_error', 'mean_squared_error'/'neg_mean_squared_error', 'median_absolute_error', 'r2'} for regressors. If a callable object or function is provided, it has to be conform with sklearn's signature scorer(estimator, X, y) ; see https://scikit-learn.org/stable/modules/generated/sklearn.metrics.make_scorer.html for more information. random_seed : int or None (default: None) Random seed for creating the test/train splits. Returns t : float The t-statistic pvalue : float Two-tailed p-value. If the chosen significance level is larger than the p-value, we reject the null hypothesis and accept that there are significant differences in the two compared models. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/paired_ttest_5x2cv/","title":"paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons"},{"location":"user_guide/evaluate/paired_ttest_5x2cv/#paired_ttest_5x2cv-5x2cv-paired-t-test-for-classifier-comparisons","text":"5x2cv paired t test procedure to compare the performance of two models from mlxtend.evaluate import paired_ttest_5x2cv","title":"paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons"},{"location":"user_guide/evaluate/paired_ttest_5x2cv/#overview","text":"The 5x2cv paired t test is a procedure for comparing the performance of two models (classifiers or regressors) that was proposed by Dietterich [1] to address shortcomings in other methods such as the resampled paired t test (see paired_ttest_resampled ) and the k-fold cross-validated paired t test (see paired_ttest_kfold_cv ). To explain how this method works, let's consider to estimator (e.g., classifiers) A and B. Further, we have a labeled dataset D . In the common hold-out method, we typically split the dataset into 2 parts: a training and a test set. In the 5x2cv paired t test, we repeat the splitting (50% training and 50% test data) 5 times. In each of the 5 iterations, we fit A and B to the training split and evaluate their performance ( p_A and p_B ) on the test split. Then, we rotate the training and test sets (the training set becomes the test set and vice versa) compute the performance again, which results in 2 performance difference measures: p^{(1)} = p^{(1)}_A - p^{(1)}_B and p^{(2)} = p^{(2)}_A - p^{(2)}_B. Then, we estimate the estimate mean and variance of the differences: \\overline{p} = \\frac{p^{(1)} + p^{(2)}}{2} and s^2 = (p^{(1)} - \\overline{p})^2 + (p^{(2)} - \\overline{p})^2. The variance of the difference is computed for the 5 iterations and then used to compute the t statistic as follows: t = \\frac{p_1^{(1)}}{\\sqrt{(1/5) \\sum_{i=1}^{5}s_i^2}}, where p_1^{(1)} is the p_1 from the very first iteration. The t statistic, assuming that it approximately follows as t distribution with 5 degrees of freedom, under the null hypothesis that the models A and B have equal performance. Using the t statistic, the p value can be computed and compared with a previously chosen significance level, e.g., \\alpha=0.05 . If the p value is smaller than \\alpha , we reject the null hypothesis and accept that there is a significant difference in the two models.","title":"Overview"},{"location":"user_guide/evaluate/paired_ttest_5x2cv/#references","text":"[1] Dietterich TG (1998) Approximate Statistical Tests for Comparing Supervised Classification Learning Algorithms. Neural Comput 10:1895\u20131923.","title":"References"},{"location":"user_guide/evaluate/paired_ttest_5x2cv/#example-1-5x2cv-paired-t-test","text":"Assume we want to compare two classification algorithms, logistic regression and a decision tree algorithm: from sklearn.linear_model import LogisticRegression from sklearn.tree import DecisionTreeClassifier from mlxtend.data import iris_data from sklearn.model_selection import train_test_split X, y = iris_data() clf1 = LogisticRegression(random_state=1) clf2 = DecisionTreeClassifier(random_state=1) X_train, X_test, y_train, y_test = \\ train_test_split(X, y, test_size=0.25, random_state=123) score1 = clf1.fit(X_train, y_train).score(X_test, y_test) score2 = clf2.fit(X_train, y_train).score(X_test, y_test) print('Logistic regression accuracy: %.2f%%' % (score1*100)) print('Decision tree accuracy: %.2f%%' % (score2*100)) Logistic regression accuracy: 97.37% Decision tree accuracy: 94.74% Note that these accuracy values are not used in the paired t test procedure as new test/train splits are generated during the resampling procedure, the values above are just serving the purpose of intuition. Now, let's assume a significance threshold of \\alpha=0.05 for rejecting the null hypothesis that both algorithms perform equally well on the dataset and conduct the 5x2cv t test: from mlxtend.evaluate import paired_ttest_5x2cv t, p = paired_ttest_5x2cv(estimator1=clf1, estimator2=clf2, X=X, y=y, random_seed=1) print('t statistic: %.3f' % t) print('p value: %.3f' % p) t statistic: -1.539 p value: 0.184 Since p > \\alpha , we cannot reject the null hypothesis and may conclude that the performance of the two algorithms is not significantly different. While it is generally not recommended to apply statistical tests multiple times without correction for multiple hypothesis testing, let us take a look at an example where the decision tree algorithm is limited to producing a very simple decision boundary that would result in a relatively bad performance: clf2 = DecisionTreeClassifier(random_state=1, max_depth=1) score2 = clf2.fit(X_train, y_train).score(X_test, y_test) print('Decision tree accuracy: %.2f%%' % (score2*100)) t, p = paired_ttest_5x2cv(estimator1=clf1, estimator2=clf2, X=X, y=y, random_seed=1) print('t statistic: %.3f' % t) print('p value: %.3f' % p) Decision tree accuracy: 63.16% t statistic: 5.386 p value: 0.003 Assuming that we conducted this test also with a significance level of \\alpha=0.05 , we can reject the null-hypothesis that both models perform equally well on this dataset, since the p-value ( p < 0.001 ) is smaller than \\alpha .","title":"Example 1 - 5x2cv paired t test"},{"location":"user_guide/evaluate/paired_ttest_5x2cv/#api","text":"paired_ttest_5x2cv(estimator1, estimator2, X, y, scoring=None, random_seed=None) Implements the 5x2cv paired t test proposed by Dieterrich (1998) to compare the performance of two models. Parameters estimator1 : scikit-learn classifier or regressor estimator2 : scikit-learn classifier or regressor X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values. scoring : str, callable, or None (default: None) If None (default), uses 'accuracy' for sklearn classifiers and 'r2' for sklearn regressors. If str, uses a sklearn scoring metric string identifier, for example {accuracy, f1, precision, recall, roc_auc} for classifiers, {'mean_absolute_error', 'mean_squared_error'/'neg_mean_squared_error', 'median_absolute_error', 'r2'} for regressors. If a callable object or function is provided, it has to be conform with sklearn's signature scorer(estimator, X, y) ; see https://scikit-learn.org/stable/modules/generated/sklearn.metrics.make_scorer.html for more information. random_seed : int or None (default: None) Random seed for creating the test/train splits. Returns t : float The t-statistic pvalue : float Two-tailed p-value. If the chosen significance level is larger than the p-value, we reject the null hypothesis and accept that there are significant differences in the two compared models. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/paired_ttest_5x2cv/","title":"API"},{"location":"user_guide/evaluate/paired_ttest_kfold_cv/","text":"paired_ttest_kfold_cv: K-fold cross-validated paired t test K-fold paired t test procedure to compare the performance of two models from mlxtend.evaluate import paired_ttest_kfold_cv Overview K-fold cross-validated paired t-test procedure is a common method for comparing the performance of two models (classifiers or regressors) and addresses some of the drawbacks of the resampled t-test procedure ; however, this method has still the problem that the training sets overlap and is not recommended to be used in practice [1], and techniques such as the paired_ttest_5x2cv should be used instead. To explain how this method works, let's consider to estimator (e.g., classifiers) A and B. Further, we have a labeled dataset D . In the common hold-out method, we typically split the dataset into 2 parts: a training and a test set. In the k-fold cross-validated paired t-test procedure, we split the test set into k parts of equal size, and each of these parts is then used for testing while the remaining k-1 parts (joined together) are used for training a classifier or regressor (i.e., the standard k-fold cross-validation procedure). In each k-fold cross-validation iteration, we then compute the difference in performance between A and B in each so that we obtain k difference measures. Now, by making the assumption that these k differences were independently drawn and follow an approximately normal distribution, we can compute the following t statistic with k-1 degrees of freedom according to Student's t test, under the null hypothesis that the models A and B have equal performance: t = \\frac{\\overline{p} \\sqrt{k}}{\\sqrt{\\sum_{i=1}^{k}(p^{(i) - \\overline{p}})^2 / (k-1)}}. Here, p^{(i)} computes the difference between the model performances in the i th iteration, p^{(i)} = p^{(i)}_A - p^{(i)}_B , and \\overline{p} represents the average difference between the classifier performances, \\overline{p} = \\frac{1}{k} \\sum^k_{i=1} p^{(i)} . Once we computed the t statistic we can compute the p value and compare it to our chosen significance level, e.g., \\alpha=0.05 . If the p value is smaller than \\alpha , we reject the null hypothesis and accept that there is a significant difference in the two models. The problem with this method, and the reason why it is not recommended to be used in practice, is that it violates an assumption of Student's t test [1]: the difference between the model performances ( p^{(i)} = p^{(i)}_A - p^{(i)}_B ) are not normal distributed because p^{(i)}_A and p^{(i)}_B are not independent the p^{(i)} 's themselves are not independent because training sets overlap References [1] Dietterich TG (1998) Approximate Statistical Tests for Comparing Supervised Classification Learning Algorithms. Neural Comput 10:1895\u20131923. Example 1 - K-fold cross-validated paired t test Assume we want to compare two classification algorithms, logistic regression and a decision tree algorithm: from sklearn.linear_model import LogisticRegression from sklearn.tree import DecisionTreeClassifier from mlxtend.data import iris_data from sklearn.model_selection import train_test_split X, y = iris_data() clf1 = LogisticRegression(random_state=1) clf2 = DecisionTreeClassifier(random_state=1) X_train, X_test, y_train, y_test = \\ train_test_split(X, y, test_size=0.25, random_state=123) score1 = clf1.fit(X_train, y_train).score(X_test, y_test) score2 = clf2.fit(X_train, y_train).score(X_test, y_test) print('Logistic regression accuracy: %.2f%%' % (score1*100)) print('Decision tree accuracy: %.2f%%' % (score2*100)) Logistic regression accuracy: 97.37% Decision tree accuracy: 94.74% Note that these accuracy values are not used in the paired t-test procedure as new test/train splits are generated during the resampling procedure, the values above are just serving the purpose of intuition. Now, let's assume a significance threshold of \\alpha=0.05 for rejecting the null hypothesis that both algorithms perform equally well on the dataset and conduct the k-fold cross-validated t-test: from mlxtend.evaluate import paired_ttest_kfold_cv t, p = paired_ttest_kfold_cv(estimator1=clf1, estimator2=clf2, X=X, y=y, random_seed=1) print('t statistic: %.3f' % t) print('p value: %.3f' % p) t statistic: -1.861 p value: 0.096 Since p > \\alpha , we cannot reject the null hypothesis and may conclude that the performance of the two algorithms is not significantly different. While it is generally not recommended to apply statistical tests multiple times without correction for multiple hypothesis testing, let us take a look at an example where the decision tree algorithm is limited to producing a very simple decision boundary that would result in a relatively bad performance: clf2 = DecisionTreeClassifier(random_state=1, max_depth=1) score2 = clf2.fit(X_train, y_train).score(X_test, y_test) print('Decision tree accuracy: %.2f%%' % (score2*100)) t, p = paired_ttest_kfold_cv(estimator1=clf1, estimator2=clf2, X=X, y=y, random_seed=1) print('t statistic: %.3f' % t) print('p value: %.3f' % p) Decision tree accuracy: 63.16% t statistic: 13.491 p value: 0.000 Assuming that we conducted this test also with a significance level of \\alpha=0.05 , we can reject the null-hypothesis that both models perform equally well on this dataset, since the p-value ( p < 0.001 ) is smaller than \\alpha . API paired_ttest_kfold_cv(estimator1, estimator2, X, y, cv=10, scoring=None, shuffle=False, random_seed=None) Implements the k-fold paired t test procedure to compare the performance of two models. Parameters estimator1 : scikit-learn classifier or regressor estimator2 : scikit-learn classifier or regressor X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values. cv : int (default: 10) Number of splits and iteration for the cross-validation procedure scoring : str, callable, or None (default: None) If None (default), uses 'accuracy' for sklearn classifiers and 'r2' for sklearn regressors. If str, uses a sklearn scoring metric string identifier, for example {accuracy, f1, precision, recall, roc_auc} for classifiers, {'mean_absolute_error', 'mean_squared_error'/'neg_mean_squared_error', 'median_absolute_error', 'r2'} for regressors. If a callable object or function is provided, it has to be conform with sklearn's signature scorer(estimator, X, y) ; see https://scikit-learn.org/stable/modules/generated/sklearn.metrics.make_scorer.html for more information. shuffle : bool (default: True) Whether to shuffle the dataset for generating the k-fold splits. random_seed : int or None (default: None) Random seed for shuffling the dataset for generating the k-fold splits. Ignored if shuffle=False. Returns t : float The t-statistic pvalue : float Two-tailed p-value. If the chosen significance level is larger than the p-value, we reject the null hypothesis and accept that there are significant differences in the two compared models. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/paired_ttest_kfold_cv/","title":"paired_ttest_kfold_cv: K-fold cross-validated paired *t* test"},{"location":"user_guide/evaluate/paired_ttest_kfold_cv/#paired_ttest_kfold_cv-k-fold-cross-validated-paired-t-test","text":"K-fold paired t test procedure to compare the performance of two models from mlxtend.evaluate import paired_ttest_kfold_cv","title":"paired_ttest_kfold_cv: K-fold cross-validated paired t test"},{"location":"user_guide/evaluate/paired_ttest_kfold_cv/#overview","text":"K-fold cross-validated paired t-test procedure is a common method for comparing the performance of two models (classifiers or regressors) and addresses some of the drawbacks of the resampled t-test procedure ; however, this method has still the problem that the training sets overlap and is not recommended to be used in practice [1], and techniques such as the paired_ttest_5x2cv should be used instead. To explain how this method works, let's consider to estimator (e.g., classifiers) A and B. Further, we have a labeled dataset D . In the common hold-out method, we typically split the dataset into 2 parts: a training and a test set. In the k-fold cross-validated paired t-test procedure, we split the test set into k parts of equal size, and each of these parts is then used for testing while the remaining k-1 parts (joined together) are used for training a classifier or regressor (i.e., the standard k-fold cross-validation procedure). In each k-fold cross-validation iteration, we then compute the difference in performance between A and B in each so that we obtain k difference measures. Now, by making the assumption that these k differences were independently drawn and follow an approximately normal distribution, we can compute the following t statistic with k-1 degrees of freedom according to Student's t test, under the null hypothesis that the models A and B have equal performance: t = \\frac{\\overline{p} \\sqrt{k}}{\\sqrt{\\sum_{i=1}^{k}(p^{(i) - \\overline{p}})^2 / (k-1)}}. Here, p^{(i)} computes the difference between the model performances in the i th iteration, p^{(i)} = p^{(i)}_A - p^{(i)}_B , and \\overline{p} represents the average difference between the classifier performances, \\overline{p} = \\frac{1}{k} \\sum^k_{i=1} p^{(i)} . Once we computed the t statistic we can compute the p value and compare it to our chosen significance level, e.g., \\alpha=0.05 . If the p value is smaller than \\alpha , we reject the null hypothesis and accept that there is a significant difference in the two models. The problem with this method, and the reason why it is not recommended to be used in practice, is that it violates an assumption of Student's t test [1]: the difference between the model performances ( p^{(i)} = p^{(i)}_A - p^{(i)}_B ) are not normal distributed because p^{(i)}_A and p^{(i)}_B are not independent the p^{(i)} 's themselves are not independent because training sets overlap","title":"Overview"},{"location":"user_guide/evaluate/paired_ttest_kfold_cv/#references","text":"[1] Dietterich TG (1998) Approximate Statistical Tests for Comparing Supervised Classification Learning Algorithms. Neural Comput 10:1895\u20131923.","title":"References"},{"location":"user_guide/evaluate/paired_ttest_kfold_cv/#example-1-k-fold-cross-validated-paired-t-test","text":"Assume we want to compare two classification algorithms, logistic regression and a decision tree algorithm: from sklearn.linear_model import LogisticRegression from sklearn.tree import DecisionTreeClassifier from mlxtend.data import iris_data from sklearn.model_selection import train_test_split X, y = iris_data() clf1 = LogisticRegression(random_state=1) clf2 = DecisionTreeClassifier(random_state=1) X_train, X_test, y_train, y_test = \\ train_test_split(X, y, test_size=0.25, random_state=123) score1 = clf1.fit(X_train, y_train).score(X_test, y_test) score2 = clf2.fit(X_train, y_train).score(X_test, y_test) print('Logistic regression accuracy: %.2f%%' % (score1*100)) print('Decision tree accuracy: %.2f%%' % (score2*100)) Logistic regression accuracy: 97.37% Decision tree accuracy: 94.74% Note that these accuracy values are not used in the paired t-test procedure as new test/train splits are generated during the resampling procedure, the values above are just serving the purpose of intuition. Now, let's assume a significance threshold of \\alpha=0.05 for rejecting the null hypothesis that both algorithms perform equally well on the dataset and conduct the k-fold cross-validated t-test: from mlxtend.evaluate import paired_ttest_kfold_cv t, p = paired_ttest_kfold_cv(estimator1=clf1, estimator2=clf2, X=X, y=y, random_seed=1) print('t statistic: %.3f' % t) print('p value: %.3f' % p) t statistic: -1.861 p value: 0.096 Since p > \\alpha , we cannot reject the null hypothesis and may conclude that the performance of the two algorithms is not significantly different. While it is generally not recommended to apply statistical tests multiple times without correction for multiple hypothesis testing, let us take a look at an example where the decision tree algorithm is limited to producing a very simple decision boundary that would result in a relatively bad performance: clf2 = DecisionTreeClassifier(random_state=1, max_depth=1) score2 = clf2.fit(X_train, y_train).score(X_test, y_test) print('Decision tree accuracy: %.2f%%' % (score2*100)) t, p = paired_ttest_kfold_cv(estimator1=clf1, estimator2=clf2, X=X, y=y, random_seed=1) print('t statistic: %.3f' % t) print('p value: %.3f' % p) Decision tree accuracy: 63.16% t statistic: 13.491 p value: 0.000 Assuming that we conducted this test also with a significance level of \\alpha=0.05 , we can reject the null-hypothesis that both models perform equally well on this dataset, since the p-value ( p < 0.001 ) is smaller than \\alpha .","title":"Example 1 - K-fold cross-validated paired t test"},{"location":"user_guide/evaluate/paired_ttest_kfold_cv/#api","text":"paired_ttest_kfold_cv(estimator1, estimator2, X, y, cv=10, scoring=None, shuffle=False, random_seed=None) Implements the k-fold paired t test procedure to compare the performance of two models. Parameters estimator1 : scikit-learn classifier or regressor estimator2 : scikit-learn classifier or regressor X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values. cv : int (default: 10) Number of splits and iteration for the cross-validation procedure scoring : str, callable, or None (default: None) If None (default), uses 'accuracy' for sklearn classifiers and 'r2' for sklearn regressors. If str, uses a sklearn scoring metric string identifier, for example {accuracy, f1, precision, recall, roc_auc} for classifiers, {'mean_absolute_error', 'mean_squared_error'/'neg_mean_squared_error', 'median_absolute_error', 'r2'} for regressors. If a callable object or function is provided, it has to be conform with sklearn's signature scorer(estimator, X, y) ; see https://scikit-learn.org/stable/modules/generated/sklearn.metrics.make_scorer.html for more information. shuffle : bool (default: True) Whether to shuffle the dataset for generating the k-fold splits. random_seed : int or None (default: None) Random seed for shuffling the dataset for generating the k-fold splits. Ignored if shuffle=False. Returns t : float The t-statistic pvalue : float Two-tailed p-value. If the chosen significance level is larger than the p-value, we reject the null hypothesis and accept that there are significant differences in the two compared models. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/paired_ttest_kfold_cv/","title":"API"},{"location":"user_guide/evaluate/paired_ttest_resampled/","text":"paired_ttest_resample: Resampled paired t test Resampled paired t test procedure to compare the performance of two models from mlxtend.evaluate import paired_ttest_resample Overview Resampled paired t test procedure (also called k-hold-out paired t test) is a popular method for comparing the performance of two models (classifiers or regressors); however, this method has many drawbacks and is not recommended to be used in practice [1], and techniques such as the paired_ttest_5x2cv should be used instead. To explain how this method works, let's consider to estimator (e.g., classifiers) A and B. Further, we have a labeled dataset D . In the common hold-out method, we typically split the dataset into 2 parts: a training and a test set. In the resampled paired t test procedure, we repeat this splitting procedure (with typically 2/3 training data and 1/3 test data) k times (usually 30). In each iteration, we train A and B on the training set and evaluate it on the test set. Then, we compute the difference in performance between A and B in each iteration so that we obtain k difference measures. Now, by making the assumption that these k differences were independently drawn and follow an approximately normal distribution, we can compute the following t statistic with k-1 degrees of freedom according to Student's t test, under the null hypothesis that the models A and B have equal performance: t = \\frac{\\overline{p} \\sqrt{k}}{\\sqrt{\\sum_{i=1}^{k}(p^{(i)} - \\overline{p})^2 / (k-1)}}. Here, p^{(i)} computes the difference between the model performances in the i th iteration, p^{(i)} = p^{(i)}_A - p^{(i)}_B , and \\overline{p} represents the average difference between the classifier performances, \\overline{p} = \\frac{1}{k} \\sum^k_{i=1} p^{(i)} . Once we computed the t statistic we can compute the p value and compare it to our chosen significance level, e.g., \\alpha=0.05 . If the p value is smaller than \\alpha , we reject the null hypothesis and accept that there is a significant difference in the two models. To summarize the procedure: i := 0 while i < k: split dataset into training and test subsets fit models A and B to the training set compute the performances of A and B on the test set record the performance difference between A and B i := i + 1 compute t-statistic compute p value from t-statistic with k-1 degrees of freedom compare p value to chosen significance threshold The problem with this method, and the reason why it is not recommended to be used in practice, is that it violates the assumptions of Student's t test [1]: the difference between the model performances ( p^{(i)} = p^{(i)}_A - p^{(i)}_B ) are not normal distributed because p^{(i)}_A and p^{(i)}_B are not independent the p^{(i)} 's themselves are not independent because of the overlapping test sets; also, test and training sets overlap as well References [1] Dietterich TG (1998) Approximate Statistical Tests for Comparing Supervised Classification Learning Algorithms. Neural Comput 10:1895\u20131923. Example 1 - Resampled paired t test Assume we want to compare two classification algorithms, logistic regression and a decision tree algorithm: from sklearn.linear_model import LogisticRegression from sklearn.tree import DecisionTreeClassifier from mlxtend.data import iris_data from sklearn.model_selection import train_test_split X, y = iris_data() clf1 = LogisticRegression(random_state=1) clf2 = DecisionTreeClassifier(random_state=1) X_train, X_test, y_train, y_test = \\ train_test_split(X, y, test_size=0.25, random_state=123) score1 = clf1.fit(X_train, y_train).score(X_test, y_test) score2 = clf2.fit(X_train, y_train).score(X_test, y_test) print('Logistic regression accuracy: %.2f%%' % (score1*100)) print('Decision tree accuracy: %.2f%%' % (score2*100)) Logistic regression accuracy: 97.37% Decision tree accuracy: 94.74% Note that these accuracy values are not used in the paired t test procedure as new test/train splits are generated during the resampling procedure, the values above are just serving the purpose of intuition. Now, let's assume a significance threshold of \\alpha=0.05 for rejecting the null hypothesis that both algorithms perform equally well on the dataset and conduct the paired sample t test: from mlxtend.evaluate import paired_ttest_resampled t, p = paired_ttest_resampled(estimator1=clf1, estimator2=clf2, X=X, y=y, random_seed=1) print('t statistic: %.3f' % t) print('p value: %.3f' % p) t statistic: -1.809 p value: 0.081 Since p > t , we cannot reject the null hypothesis and may conclude that the performance of the two algorithms is not significantly different. While it is generally not recommended to apply statistical tests multiple times without correction for multiple hypothesis testing, let us take a look at an example where the decision tree algorithm is limited to producing a very simple decision boundary that would result in a relatively bad performance: clf2 = DecisionTreeClassifier(random_state=1, max_depth=1) score2 = clf2.fit(X_train, y_train).score(X_test, y_test) print('Decision tree accuracy: %.2f%%' % (score2*100)) t, p = paired_ttest_resampled(estimator1=clf1, estimator2=clf2, X=X, y=y, random_seed=1) print('t statistic: %.3f' % t) print('p value: %.3f' % p) Decision tree accuracy: 63.16% t statistic: 39.214 p value: 0.000 Assuming that we conducted this test also with a significance level of \\alpha=0.05 , we can reject the null-hypothesis that both models perform equally well on this dataset, since the p-value ( p < 0.001 ) is smaller than \\alpha . API paired_ttest_resampled(estimator1, estimator2, X, y, num_rounds=30, test_size=0.3, scoring=None, random_seed=None) Implements the resampled paired t test procedure to compare the performance of two models (also called k-hold-out paired t test). Parameters estimator1 : scikit-learn classifier or regressor estimator2 : scikit-learn classifier or regressor X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values. num_rounds : int (default: 30) Number of resampling iterations (i.e., train/test splits) test_size : float or int (default: 0.3) If float, should be between 0.0 and 1.0 and represent the proportion of the dataset to use as a test set. If int, represents the absolute number of test exsamples. scoring : str, callable, or None (default: None) If None (default), uses 'accuracy' for sklearn classifiers and 'r2' for sklearn regressors. If str, uses a sklearn scoring metric string identifier, for example {accuracy, f1, precision, recall, roc_auc} for classifiers, {'mean_absolute_error', 'mean_squared_error'/'neg_mean_squared_error', 'median_absolute_error', 'r2'} for regressors. If a callable object or function is provided, it has to be conform with sklearn's signature scorer(estimator, X, y) ; see https://scikit-learn.org/stable/modules/generated/sklearn.metrics.make_scorer.html for more information. random_seed : int or None (default: None) Random seed for creating the test/train splits. Returns t : float The t-statistic pvalue : float Two-tailed p-value. If the chosen significance level is larger than the p-value, we reject the null hypothesis and accept that there are significant differences in the two compared models. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/paired_ttest_resampled/","title":"paired_ttest_resample: Resampled paired *t* test"},{"location":"user_guide/evaluate/paired_ttest_resampled/#paired_ttest_resample-resampled-paired-t-test","text":"Resampled paired t test procedure to compare the performance of two models from mlxtend.evaluate import paired_ttest_resample","title":"paired_ttest_resample: Resampled paired t test"},{"location":"user_guide/evaluate/paired_ttest_resampled/#overview","text":"Resampled paired t test procedure (also called k-hold-out paired t test) is a popular method for comparing the performance of two models (classifiers or regressors); however, this method has many drawbacks and is not recommended to be used in practice [1], and techniques such as the paired_ttest_5x2cv should be used instead. To explain how this method works, let's consider to estimator (e.g., classifiers) A and B. Further, we have a labeled dataset D . In the common hold-out method, we typically split the dataset into 2 parts: a training and a test set. In the resampled paired t test procedure, we repeat this splitting procedure (with typically 2/3 training data and 1/3 test data) k times (usually 30). In each iteration, we train A and B on the training set and evaluate it on the test set. Then, we compute the difference in performance between A and B in each iteration so that we obtain k difference measures. Now, by making the assumption that these k differences were independently drawn and follow an approximately normal distribution, we can compute the following t statistic with k-1 degrees of freedom according to Student's t test, under the null hypothesis that the models A and B have equal performance: t = \\frac{\\overline{p} \\sqrt{k}}{\\sqrt{\\sum_{i=1}^{k}(p^{(i)} - \\overline{p})^2 / (k-1)}}. Here, p^{(i)} computes the difference between the model performances in the i th iteration, p^{(i)} = p^{(i)}_A - p^{(i)}_B , and \\overline{p} represents the average difference between the classifier performances, \\overline{p} = \\frac{1}{k} \\sum^k_{i=1} p^{(i)} . Once we computed the t statistic we can compute the p value and compare it to our chosen significance level, e.g., \\alpha=0.05 . If the p value is smaller than \\alpha , we reject the null hypothesis and accept that there is a significant difference in the two models. To summarize the procedure: i := 0 while i < k: split dataset into training and test subsets fit models A and B to the training set compute the performances of A and B on the test set record the performance difference between A and B i := i + 1 compute t-statistic compute p value from t-statistic with k-1 degrees of freedom compare p value to chosen significance threshold The problem with this method, and the reason why it is not recommended to be used in practice, is that it violates the assumptions of Student's t test [1]: the difference between the model performances ( p^{(i)} = p^{(i)}_A - p^{(i)}_B ) are not normal distributed because p^{(i)}_A and p^{(i)}_B are not independent the p^{(i)} 's themselves are not independent because of the overlapping test sets; also, test and training sets overlap as well","title":"Overview"},{"location":"user_guide/evaluate/paired_ttest_resampled/#references","text":"[1] Dietterich TG (1998) Approximate Statistical Tests for Comparing Supervised Classification Learning Algorithms. Neural Comput 10:1895\u20131923.","title":"References"},{"location":"user_guide/evaluate/paired_ttest_resampled/#example-1-resampled-paired-t-test","text":"Assume we want to compare two classification algorithms, logistic regression and a decision tree algorithm: from sklearn.linear_model import LogisticRegression from sklearn.tree import DecisionTreeClassifier from mlxtend.data import iris_data from sklearn.model_selection import train_test_split X, y = iris_data() clf1 = LogisticRegression(random_state=1) clf2 = DecisionTreeClassifier(random_state=1) X_train, X_test, y_train, y_test = \\ train_test_split(X, y, test_size=0.25, random_state=123) score1 = clf1.fit(X_train, y_train).score(X_test, y_test) score2 = clf2.fit(X_train, y_train).score(X_test, y_test) print('Logistic regression accuracy: %.2f%%' % (score1*100)) print('Decision tree accuracy: %.2f%%' % (score2*100)) Logistic regression accuracy: 97.37% Decision tree accuracy: 94.74% Note that these accuracy values are not used in the paired t test procedure as new test/train splits are generated during the resampling procedure, the values above are just serving the purpose of intuition. Now, let's assume a significance threshold of \\alpha=0.05 for rejecting the null hypothesis that both algorithms perform equally well on the dataset and conduct the paired sample t test: from mlxtend.evaluate import paired_ttest_resampled t, p = paired_ttest_resampled(estimator1=clf1, estimator2=clf2, X=X, y=y, random_seed=1) print('t statistic: %.3f' % t) print('p value: %.3f' % p) t statistic: -1.809 p value: 0.081 Since p > t , we cannot reject the null hypothesis and may conclude that the performance of the two algorithms is not significantly different. While it is generally not recommended to apply statistical tests multiple times without correction for multiple hypothesis testing, let us take a look at an example where the decision tree algorithm is limited to producing a very simple decision boundary that would result in a relatively bad performance: clf2 = DecisionTreeClassifier(random_state=1, max_depth=1) score2 = clf2.fit(X_train, y_train).score(X_test, y_test) print('Decision tree accuracy: %.2f%%' % (score2*100)) t, p = paired_ttest_resampled(estimator1=clf1, estimator2=clf2, X=X, y=y, random_seed=1) print('t statistic: %.3f' % t) print('p value: %.3f' % p) Decision tree accuracy: 63.16% t statistic: 39.214 p value: 0.000 Assuming that we conducted this test also with a significance level of \\alpha=0.05 , we can reject the null-hypothesis that both models perform equally well on this dataset, since the p-value ( p < 0.001 ) is smaller than \\alpha .","title":"Example 1 - Resampled paired t test"},{"location":"user_guide/evaluate/paired_ttest_resampled/#api","text":"paired_ttest_resampled(estimator1, estimator2, X, y, num_rounds=30, test_size=0.3, scoring=None, random_seed=None) Implements the resampled paired t test procedure to compare the performance of two models (also called k-hold-out paired t test). Parameters estimator1 : scikit-learn classifier or regressor estimator2 : scikit-learn classifier or regressor X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values. num_rounds : int (default: 30) Number of resampling iterations (i.e., train/test splits) test_size : float or int (default: 0.3) If float, should be between 0.0 and 1.0 and represent the proportion of the dataset to use as a test set. If int, represents the absolute number of test exsamples. scoring : str, callable, or None (default: None) If None (default), uses 'accuracy' for sklearn classifiers and 'r2' for sklearn regressors. If str, uses a sklearn scoring metric string identifier, for example {accuracy, f1, precision, recall, roc_auc} for classifiers, {'mean_absolute_error', 'mean_squared_error'/'neg_mean_squared_error', 'median_absolute_error', 'r2'} for regressors. If a callable object or function is provided, it has to be conform with sklearn's signature scorer(estimator, X, y) ; see https://scikit-learn.org/stable/modules/generated/sklearn.metrics.make_scorer.html for more information. random_seed : int or None (default: None) Random seed for creating the test/train splits. Returns t : float The t-statistic pvalue : float Two-tailed p-value. If the chosen significance level is larger than the p-value, we reject the null hypothesis and accept that there are significant differences in the two compared models. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/paired_ttest_resampled/","title":"API"},{"location":"user_guide/evaluate/permutation_test/","text":"permutation_test: Permutation test for hypothesis testing An implementation of a permutation test for hypothesis testing -- testing the null hypothesis that two different groups come from the same distribution. from mlxtend.evaluate import permutation_test Overview Permutation tests (also called exact tests, randomization tests, or re-randomization tests) are nonparametric test procedures to test the null hypothesis that two different groups come from the same distribution. A permutation test can be used for significance or hypothesis testing (including A/B testing) without requiring to make any assumptions about the sampling distribution (e.g., it doesn't require the samples to be normal distributed). In this document, we will refer to the exact method as \"permutation test\" and the approximated method as \"randomization test.\" Permutation Test Mechanics Under the null hypothesis (treatment = control), any permutations are equally likely. (Note that there are (n+m)! permutations, where n is the number of records in the treatment sample, and m is the number of records in the control sample). For a two-sided test, we define the alternative hypothesis that the two samples are different (e.g., treatment != control). Compute the difference (here: mean) of sample x and sample y Combine all measurements into a single dataset Draw a permuted dataset from all possible permutations of the dataset in 2. Divide the permuted dataset into two datasets x' and y' of size n and m , respectively Compute the difference (here: mean) of sample x' and sample y' and record this difference Repeat steps 3-5 until all permutations are evaluated Return the p-value as the number of times the recorded differences were at least as extreme as the original difference from 1. and divide this number by the total number of permutations Here, the p-value is defined as the probability, given the null hypothesis (no difference between the samples) is true, that we obtain results that are at least as extreme as the results we observed (i.e., the sample difference from 1.). More formally, we can express the computation of the p-value as follows (adapted from [2]): p(t \\geq t_0) = \\frac{1}{(n+m)!} \\sum^{(n+m)!}_{j=1} I(t_j \\geq t_0), where t_0 is the observed value of the test statistic (1. in the list above), and t is the t-value, the statistic computed from the resamples (5.) t(x'_1, x'_2, ..., x'_n, y'_1, y'_2, ..., x'_m) = |\\bar{x'} - \\bar{y'}| , and I is the indicator function. Given a significance level that we specify prior to carrying out the permutation test (e.g., alpha=0.05), we fail to reject the null hypothesis if the p-value is greater than alpha. Note that if the number of permutation is large, sampling all permutation may not computationally be feasible. Thus, a common approximation is to perfom k rounds of permutations (where k is typically a value between 1000 and 2000). Paired Samples The permutation (/randomization) tests can also be performed for paired samples by setting paired=True . The paired tests are related to the regular permutation test procedure described above except that the permuted samples are created by randomly swapping the a treatment and a control data point within each pair. References [1] Efron, Bradley and Tibshirani, R. J., An introduction to the bootstrap, Chapman & Hall/CRC Monographs on Statistics & Applied Probability, 1994. [2] Unpingco, Jos\u00e9. Python for probability, statistics, and machine learning. Springer, 2016. [3] Pitman, E. J. G., Significance tests which may be applied to samples from any population, Royal Statistical Society Supplement, 1937, 4: 119-30 and 225-32 Example 1 -- Two-sided randomization test Perform a two-sided randomization test to test the null hypothesis that two groups, \"treatment\" and \"control\" come from the same distribution. We specify alpha=0.01 as our significance level. treatment = [ 28.44, 29.32, 31.22, 29.58, 30.34, 28.76, 29.21, 30.4 , 31.12, 31.78, 27.58, 31.57, 30.73, 30.43, 30.31, 30.32, 29.18, 29.52, 29.22, 30.56] control = [ 33.51, 30.63, 32.38, 32.52, 29.41, 30.93, 49.78, 28.96, 35.77, 31.42, 30.76, 30.6 , 23.64, 30.54, 47.78, 31.98, 34.52, 32.42, 31.32, 40.72] Since evaluating all possible permutations may take a while, we will use the approximation method (see the introduction for details) i.e., randomization test: from mlxtend.evaluate import permutation_test p_value = permutation_test(treatment, control, method='approximate', num_rounds=10000, seed=0) print(p_value) 0.0066993300669933005 Since p-value < alpha, we can reject the null hypothesis that the two samples come from the same distribution. Example 2 -- Permutation test for calculating the p-value for correlation analysis (Pearson's R) Note: this is a one-sided hypothesis testing as we conduct the permutation test as \"how many times obtain a correlation coefficient that is greater than the observed value?\" import numpy as np from mlxtend.evaluate import permutation_test x = np.array([1, 2, 3, 4, 5, 6]) y = np.array([2, 4, 1, 5, 6, 7]) print('Observed pearson R: %.2f' % np.corrcoef(x, y)[1][0]) p_value = permutation_test(x, y, method='exact', func=lambda x, y: np.corrcoef(x, y)[1][0], seed=0) print('P value: %.2f' % p_value) Observed pearson R: 0.81 P value: 0.10 Example 3 -- Paired two-sample randomization test Suppose we have a dataset consisting of the the depths (in meters) of seven lakes of Wisconsin: \\begin{array}{cccccccc} \\text { Year } & \\text{ Lake } 1 & 2 & 3 & 4 & 5 & 6 & 7 \\\\ \\hline 1980: & 3.67 & 1.72 & 3.46 & 2.60 & 2.03 & 2.10 & 3.01 \\\\ \\hline 1990: & 2.11 & 1.79 & 2.71 & 1.89 & 1.69 & 1.71 & 2.01 \\\\ \\hline \\end{array} We are interested in testing the null hypothesis that the lakes in 1980 and 1990 don't have a significantly different depth. For this paired two-sample test, we are conducting a randomization test for paired samples at a significance level of 0.05: from mlxtend.evaluate import permutation_test lakes_1980 = [3.67, 1.72, 3.46, 2.60, 2.03, 2.10, 3.01] lakes_1990 = [2.11, 1.79, 2.71, 1.89, 1.69, 1.71, 2.01] p_value = permutation_test( lakes_1980, lakes_1990, paired=True, method=\"approximate\", seed=0, num_rounds=100000 ) print('P value: %.3f' % p_value) P value: 0.031 Since the p value is smaller than the significance threshold of 0.05, we conclude that there is a significant difference between the lake depths in 1980 and 1990. API permutation_test(x, y, func='x_mean != y_mean', method='exact', num_rounds=1000, seed=None, paired=False) Nonparametric permutation test Parameters x : list or numpy array with shape (n_datapoints,) A list or 1D numpy array of the first sample (e.g., the treatment group). y : list or numpy array with shape (n_datapoints,) A list or 1D numpy array of the second sample (e.g., the control group). func : custom function or str (default: 'x_mean != y_mean') function to compute the statistic for the permutation test. - If 'x_mean != y_mean', uses func=lambda x, y: np.abs(np.mean(x) - np.mean(y))) for a two-sided test. - If 'x_mean > y_mean', uses func=lambda x, y: np.mean(x) - np.mean(y)) for a one-sided test. - If 'x_mean < y_mean', uses func=lambda x, y: np.mean(y) - np.mean(x)) for a one-sided test. method : 'approximate' or 'exact' (default: 'exact') If 'exact' (default), all possible permutations are considered. If 'approximate' the number of drawn samples is given by num_rounds . Note that 'exact' is typically not feasible unless the dataset size is relatively small. paired : bool If True, a paired test is performed by only exchanging each datapoint with its associate. num_rounds : int (default: 1000) The number of permutation samples if method='approximate' . seed : int or None (default: None) The random seed for generating permutation samples if method='approximate' . Returns p-value under the null hypothesis Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/permutation_test/","title":"permutation_test: Permutation test for hypothesis testing"},{"location":"user_guide/evaluate/permutation_test/#permutation_test-permutation-test-for-hypothesis-testing","text":"An implementation of a permutation test for hypothesis testing -- testing the null hypothesis that two different groups come from the same distribution. from mlxtend.evaluate import permutation_test","title":"permutation_test: Permutation test for hypothesis testing"},{"location":"user_guide/evaluate/permutation_test/#overview","text":"Permutation tests (also called exact tests, randomization tests, or re-randomization tests) are nonparametric test procedures to test the null hypothesis that two different groups come from the same distribution. A permutation test can be used for significance or hypothesis testing (including A/B testing) without requiring to make any assumptions about the sampling distribution (e.g., it doesn't require the samples to be normal distributed). In this document, we will refer to the exact method as \"permutation test\" and the approximated method as \"randomization test.\"","title":"Overview"},{"location":"user_guide/evaluate/permutation_test/#permutation-test-mechanics","text":"Under the null hypothesis (treatment = control), any permutations are equally likely. (Note that there are (n+m)! permutations, where n is the number of records in the treatment sample, and m is the number of records in the control sample). For a two-sided test, we define the alternative hypothesis that the two samples are different (e.g., treatment != control). Compute the difference (here: mean) of sample x and sample y Combine all measurements into a single dataset Draw a permuted dataset from all possible permutations of the dataset in 2. Divide the permuted dataset into two datasets x' and y' of size n and m , respectively Compute the difference (here: mean) of sample x' and sample y' and record this difference Repeat steps 3-5 until all permutations are evaluated Return the p-value as the number of times the recorded differences were at least as extreme as the original difference from 1. and divide this number by the total number of permutations Here, the p-value is defined as the probability, given the null hypothesis (no difference between the samples) is true, that we obtain results that are at least as extreme as the results we observed (i.e., the sample difference from 1.). More formally, we can express the computation of the p-value as follows (adapted from [2]): p(t \\geq t_0) = \\frac{1}{(n+m)!} \\sum^{(n+m)!}_{j=1} I(t_j \\geq t_0), where t_0 is the observed value of the test statistic (1. in the list above), and t is the t-value, the statistic computed from the resamples (5.) t(x'_1, x'_2, ..., x'_n, y'_1, y'_2, ..., x'_m) = |\\bar{x'} - \\bar{y'}| , and I is the indicator function. Given a significance level that we specify prior to carrying out the permutation test (e.g., alpha=0.05), we fail to reject the null hypothesis if the p-value is greater than alpha. Note that if the number of permutation is large, sampling all permutation may not computationally be feasible. Thus, a common approximation is to perfom k rounds of permutations (where k is typically a value between 1000 and 2000).","title":"Permutation Test Mechanics"},{"location":"user_guide/evaluate/permutation_test/#paired-samples","text":"The permutation (/randomization) tests can also be performed for paired samples by setting paired=True . The paired tests are related to the regular permutation test procedure described above except that the permuted samples are created by randomly swapping the a treatment and a control data point within each pair.","title":"Paired Samples"},{"location":"user_guide/evaluate/permutation_test/#references","text":"[1] Efron, Bradley and Tibshirani, R. J., An introduction to the bootstrap, Chapman & Hall/CRC Monographs on Statistics & Applied Probability, 1994. [2] Unpingco, Jos\u00e9. Python for probability, statistics, and machine learning. Springer, 2016. [3] Pitman, E. J. G., Significance tests which may be applied to samples from any population, Royal Statistical Society Supplement, 1937, 4: 119-30 and 225-32","title":"References"},{"location":"user_guide/evaluate/permutation_test/#example-1-two-sided-randomization-test","text":"Perform a two-sided randomization test to test the null hypothesis that two groups, \"treatment\" and \"control\" come from the same distribution. We specify alpha=0.01 as our significance level. treatment = [ 28.44, 29.32, 31.22, 29.58, 30.34, 28.76, 29.21, 30.4 , 31.12, 31.78, 27.58, 31.57, 30.73, 30.43, 30.31, 30.32, 29.18, 29.52, 29.22, 30.56] control = [ 33.51, 30.63, 32.38, 32.52, 29.41, 30.93, 49.78, 28.96, 35.77, 31.42, 30.76, 30.6 , 23.64, 30.54, 47.78, 31.98, 34.52, 32.42, 31.32, 40.72] Since evaluating all possible permutations may take a while, we will use the approximation method (see the introduction for details) i.e., randomization test: from mlxtend.evaluate import permutation_test p_value = permutation_test(treatment, control, method='approximate', num_rounds=10000, seed=0) print(p_value) 0.0066993300669933005 Since p-value < alpha, we can reject the null hypothesis that the two samples come from the same distribution.","title":"Example 1 -- Two-sided randomization test"},{"location":"user_guide/evaluate/permutation_test/#example-2-permutation-test-for-calculating-the-p-value-for-correlation-analysis-pearsons-r","text":"Note: this is a one-sided hypothesis testing as we conduct the permutation test as \"how many times obtain a correlation coefficient that is greater than the observed value?\" import numpy as np from mlxtend.evaluate import permutation_test x = np.array([1, 2, 3, 4, 5, 6]) y = np.array([2, 4, 1, 5, 6, 7]) print('Observed pearson R: %.2f' % np.corrcoef(x, y)[1][0]) p_value = permutation_test(x, y, method='exact', func=lambda x, y: np.corrcoef(x, y)[1][0], seed=0) print('P value: %.2f' % p_value) Observed pearson R: 0.81 P value: 0.10","title":"Example 2 -- Permutation test for calculating the p-value for correlation analysis (Pearson's R)"},{"location":"user_guide/evaluate/permutation_test/#example-3-paired-two-sample-randomization-test","text":"Suppose we have a dataset consisting of the the depths (in meters) of seven lakes of Wisconsin: \\begin{array}{cccccccc} \\text { Year } & \\text{ Lake } 1 & 2 & 3 & 4 & 5 & 6 & 7 \\\\ \\hline 1980: & 3.67 & 1.72 & 3.46 & 2.60 & 2.03 & 2.10 & 3.01 \\\\ \\hline 1990: & 2.11 & 1.79 & 2.71 & 1.89 & 1.69 & 1.71 & 2.01 \\\\ \\hline \\end{array} We are interested in testing the null hypothesis that the lakes in 1980 and 1990 don't have a significantly different depth. For this paired two-sample test, we are conducting a randomization test for paired samples at a significance level of 0.05: from mlxtend.evaluate import permutation_test lakes_1980 = [3.67, 1.72, 3.46, 2.60, 2.03, 2.10, 3.01] lakes_1990 = [2.11, 1.79, 2.71, 1.89, 1.69, 1.71, 2.01] p_value = permutation_test( lakes_1980, lakes_1990, paired=True, method=\"approximate\", seed=0, num_rounds=100000 ) print('P value: %.3f' % p_value) P value: 0.031 Since the p value is smaller than the significance threshold of 0.05, we conclude that there is a significant difference between the lake depths in 1980 and 1990.","title":"Example 3 -- Paired two-sample randomization test"},{"location":"user_guide/evaluate/permutation_test/#api","text":"permutation_test(x, y, func='x_mean != y_mean', method='exact', num_rounds=1000, seed=None, paired=False) Nonparametric permutation test Parameters x : list or numpy array with shape (n_datapoints,) A list or 1D numpy array of the first sample (e.g., the treatment group). y : list or numpy array with shape (n_datapoints,) A list or 1D numpy array of the second sample (e.g., the control group). func : custom function or str (default: 'x_mean != y_mean') function to compute the statistic for the permutation test. - If 'x_mean != y_mean', uses func=lambda x, y: np.abs(np.mean(x) - np.mean(y))) for a two-sided test. - If 'x_mean > y_mean', uses func=lambda x, y: np.mean(x) - np.mean(y)) for a one-sided test. - If 'x_mean < y_mean', uses func=lambda x, y: np.mean(y) - np.mean(x)) for a one-sided test. method : 'approximate' or 'exact' (default: 'exact') If 'exact' (default), all possible permutations are considered. If 'approximate' the number of drawn samples is given by num_rounds . Note that 'exact' is typically not feasible unless the dataset size is relatively small. paired : bool If True, a paired test is performed by only exchanging each datapoint with its associate. num_rounds : int (default: 1000) The number of permutation samples if method='approximate' . seed : int or None (default: None) The random seed for generating permutation samples if method='approximate' . Returns p-value under the null hypothesis Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/permutation_test/","title":"API"},{"location":"user_guide/evaluate/proportion_difference/","text":"proportion_difference: Difference of proportions test for classifiers. Test of the difference of proportions for classifier performance comparison. from mlxtend.evaluate import proportion_difference Overview There are several different statistical hypothesis testing frameworks that are being used in practice to compare the performance of classification models, including common methods such as difference of two proportions (here, the proportions are the estimated generalization accuracies from a test set), for which we can construct 95% confidence intervals based on the concept of the Normal Approximation to the Binomial that was covered in Part I. Performing a z-score test for two population proportions is inarguably the most straight-forward way to compare to models (but certainly not the best!): In a nutshell, if the 95% confidence intervals of the accuracies of two models do not overlap, we can reject the null hypothesis that the performance of both classifiers is equal at a confidence level of \\alpha=0.05 (or 5% probability). Violations of assumptions aside (for instance that the test set samples are not independent), as Thomas Dietterich noted based on empircal results in a simulated study [1], this test tends to have a high false positive rate (here: incorrectly detecting difference when there is none), which is among the reasons why it is not recommended in practice. Nonetheless, for the sake of completeness, and since it a commonly used method in practice, the general procedure is outlined below as follows (which also generally applies to the different hypothesis tests presented later): formulate the hypothesis to be tested (for instance, the null hypothesis stating that the proportions are the same; consequently, the alternative hypothesis that the proportions are different, if we use a two-tailed test); decide upon a significance threshold (for instance, if the probability of observing a difference more extreme than the one observed is more than 5%, then we plan to reject the null hypothesis); analyze the data, compute the test statistic (here: z-score), and compare its associated p-value (probability) to the previously determined significance threshold; based on the p-value and significance threshold, either accept or reject the null hypothesis at the given confidence level and interpret the results. The z-score is computed as the observed difference divided by the square root for their combined variances z = \\frac{ACC_1 - ACC_2}{\\sqrt{\\sigma_{1}^2 + \\sigma_{2}^2}}, where ACC_1 is the accuracy of one model and ACC_2 is the accuracy of a second model estimated from the test set. Recall that we computed the variance of the estimated of the estimated accuracy as \\sigma^2 = \\frac{ACC(1-ACC)}{n} in Part I and then computed the confidence interval (Normal Approximation Interval) as ACC \\pm z \\times \\sigma, where z=1.96 for a 95% confidence interval. Comparing the confidence intervals of two accuracy estimates and checking whether they overlap is then analogous to computing the z value for the difference in proportions and comparing the probability (p-value) to the chosen significance threshold. So, to compute the z-score directly for the difference of two proportions, ACC_1 and ACC_2 , we pool these proportions (assuming that ACC_1 and ACC_2 are the performances of two models estimated on two indendent test sets of size n_1 and n_2 , respectively), ACC_{1, 2} = \\frac{ACC_1 \\times n_1 + ACC_2 \\times n_2}{n_1 + n_2}, and compute the standard deviation as \\sigma_{1,2} = \\sqrt{\\frac{ACC_{1, 2} (1 - ACC_{1, 2})}{n_1 + n_2}}, such that we can compute the z-score, z = \\frac{ACC_1 - ACC_2}{\\sigma_{1,2}}. Since, due to using the same test set (and violating the independence assumption) we have n_1 = n_2 = n , so that we can simplify the z-score computation to z = \\frac{ACC_1 - ACC_2}{\\sqrt{2\\sigma^2}} = \\frac{ACC_1 - ACC_2}{\\sqrt{2\\cdot ACC_{1,2}(1-ACC_{1,2}))/n}}. where ACC_{1, 2} is simply (ACC_1 + ACC_2)/2 . In the second step, based on the computed z value (this assumes the the test errors are independent, which is usually violated in practice as we use the same test set) we can reject the null hypothesis that the a pair of models has equal performance (here, measured in \"classification aaccuracy\") at an \\alpha=0.05 level if z is greater than 1.96. Or if we want to put in the extra work, we can compute the area under the a standard normal cumulative distribution at the z-score threshold. If we find this p-value is smaller than a significance level we set prior to conducting the test, then we can reject the null hypothesis at that given significance level. The problem with this test though is that we use the same test set to compute the accuracy of the two classifiers; thus, it might be better to use a paired test such as a paired sample t-test, but a more robust alternative is the McNemar test. References [1] Dietterich, Thomas G. \"Approximate statistical tests for comparing supervised classification learning algorithms.\" Neural computation 10, no. 7 (1998): 1895-1923. Example 1 - Difference of Proportions As an example for applying this test, consider the following 2 model predictions: import numpy as np ## Dataset: # ground truth labels of the test dataset: y_true = np.array([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]) # predictions by 3 classifiers (`y_model_1`, `y_model_2`, and `y_model_3`): y_model_1 = np.array([1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]) y_model_2 = np.array([1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]) Assume, the test accuracies are as follows: acc_1 = np.sum(y_true == y_model_1) / y_true.shape[0] acc_2 = np.sum(y_true == y_model_2) / y_true.shape[0] print('Accuracy Model 1:', acc_1) print('Accuracy Model 2:', acc_2) Accuracy Model 1: 0.84 Accuracy Model 2: 0.92 Now, setting a significance threshold of \\alpha=0.05 and conducting the test from mlxtend.evaluate import proportion_difference z, p_value = proportion_difference(acc_1, acc_2, n_1=y_true.shape[0]) print('z: %.3f' % z) print('p-value: %.3f' % p_value) z: -1.754 p-value: 0.040 we find that there is a statistically significant difference between the model performances. It should be highlighted though that using this test, due to the typical independence violation of using the same test set as well as its high false positive rate, it is not recommended to use this test in practice. API proportion_difference(proportion_1, proportion_2, n_1, n_2=None) Computes the test statistic and p-value for a difference of proportions test. Parameters proportion_1 : float The first proportion proportion_2 : float The second proportion n_1 : int The sample size of the first test sample n_2 : int or None (default=None) The sample size of the second test sample. If None , n_1 = n_2 . Returns z, p : float or None, float Returns the z-score and the p-value Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/proportion_difference/","title":"proportion_difference: Difference of proportions test for classifiers."},{"location":"user_guide/evaluate/proportion_difference/#proportion_difference-difference-of-proportions-test-for-classifiers","text":"Test of the difference of proportions for classifier performance comparison. from mlxtend.evaluate import proportion_difference","title":"proportion_difference: Difference of proportions test for classifiers."},{"location":"user_guide/evaluate/proportion_difference/#overview","text":"There are several different statistical hypothesis testing frameworks that are being used in practice to compare the performance of classification models, including common methods such as difference of two proportions (here, the proportions are the estimated generalization accuracies from a test set), for which we can construct 95% confidence intervals based on the concept of the Normal Approximation to the Binomial that was covered in Part I. Performing a z-score test for two population proportions is inarguably the most straight-forward way to compare to models (but certainly not the best!): In a nutshell, if the 95% confidence intervals of the accuracies of two models do not overlap, we can reject the null hypothesis that the performance of both classifiers is equal at a confidence level of \\alpha=0.05 (or 5% probability). Violations of assumptions aside (for instance that the test set samples are not independent), as Thomas Dietterich noted based on empircal results in a simulated study [1], this test tends to have a high false positive rate (here: incorrectly detecting difference when there is none), which is among the reasons why it is not recommended in practice. Nonetheless, for the sake of completeness, and since it a commonly used method in practice, the general procedure is outlined below as follows (which also generally applies to the different hypothesis tests presented later): formulate the hypothesis to be tested (for instance, the null hypothesis stating that the proportions are the same; consequently, the alternative hypothesis that the proportions are different, if we use a two-tailed test); decide upon a significance threshold (for instance, if the probability of observing a difference more extreme than the one observed is more than 5%, then we plan to reject the null hypothesis); analyze the data, compute the test statistic (here: z-score), and compare its associated p-value (probability) to the previously determined significance threshold; based on the p-value and significance threshold, either accept or reject the null hypothesis at the given confidence level and interpret the results. The z-score is computed as the observed difference divided by the square root for their combined variances z = \\frac{ACC_1 - ACC_2}{\\sqrt{\\sigma_{1}^2 + \\sigma_{2}^2}}, where ACC_1 is the accuracy of one model and ACC_2 is the accuracy of a second model estimated from the test set. Recall that we computed the variance of the estimated of the estimated accuracy as \\sigma^2 = \\frac{ACC(1-ACC)}{n} in Part I and then computed the confidence interval (Normal Approximation Interval) as ACC \\pm z \\times \\sigma, where z=1.96 for a 95% confidence interval. Comparing the confidence intervals of two accuracy estimates and checking whether they overlap is then analogous to computing the z value for the difference in proportions and comparing the probability (p-value) to the chosen significance threshold. So, to compute the z-score directly for the difference of two proportions, ACC_1 and ACC_2 , we pool these proportions (assuming that ACC_1 and ACC_2 are the performances of two models estimated on two indendent test sets of size n_1 and n_2 , respectively), ACC_{1, 2} = \\frac{ACC_1 \\times n_1 + ACC_2 \\times n_2}{n_1 + n_2}, and compute the standard deviation as \\sigma_{1,2} = \\sqrt{\\frac{ACC_{1, 2} (1 - ACC_{1, 2})}{n_1 + n_2}}, such that we can compute the z-score, z = \\frac{ACC_1 - ACC_2}{\\sigma_{1,2}}. Since, due to using the same test set (and violating the independence assumption) we have n_1 = n_2 = n , so that we can simplify the z-score computation to z = \\frac{ACC_1 - ACC_2}{\\sqrt{2\\sigma^2}} = \\frac{ACC_1 - ACC_2}{\\sqrt{2\\cdot ACC_{1,2}(1-ACC_{1,2}))/n}}. where ACC_{1, 2} is simply (ACC_1 + ACC_2)/2 . In the second step, based on the computed z value (this assumes the the test errors are independent, which is usually violated in practice as we use the same test set) we can reject the null hypothesis that the a pair of models has equal performance (here, measured in \"classification aaccuracy\") at an \\alpha=0.05 level if z is greater than 1.96. Or if we want to put in the extra work, we can compute the area under the a standard normal cumulative distribution at the z-score threshold. If we find this p-value is smaller than a significance level we set prior to conducting the test, then we can reject the null hypothesis at that given significance level. The problem with this test though is that we use the same test set to compute the accuracy of the two classifiers; thus, it might be better to use a paired test such as a paired sample t-test, but a more robust alternative is the McNemar test.","title":"Overview"},{"location":"user_guide/evaluate/proportion_difference/#references","text":"[1] Dietterich, Thomas G. \"Approximate statistical tests for comparing supervised classification learning algorithms.\" Neural computation 10, no. 7 (1998): 1895-1923.","title":"References"},{"location":"user_guide/evaluate/proportion_difference/#example-1-difference-of-proportions","text":"As an example for applying this test, consider the following 2 model predictions: import numpy as np ## Dataset: # ground truth labels of the test dataset: y_true = np.array([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]) # predictions by 3 classifiers (`y_model_1`, `y_model_2`, and `y_model_3`): y_model_1 = np.array([1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]) y_model_2 = np.array([1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]) Assume, the test accuracies are as follows: acc_1 = np.sum(y_true == y_model_1) / y_true.shape[0] acc_2 = np.sum(y_true == y_model_2) / y_true.shape[0] print('Accuracy Model 1:', acc_1) print('Accuracy Model 2:', acc_2) Accuracy Model 1: 0.84 Accuracy Model 2: 0.92 Now, setting a significance threshold of \\alpha=0.05 and conducting the test from mlxtend.evaluate import proportion_difference z, p_value = proportion_difference(acc_1, acc_2, n_1=y_true.shape[0]) print('z: %.3f' % z) print('p-value: %.3f' % p_value) z: -1.754 p-value: 0.040 we find that there is a statistically significant difference between the model performances. It should be highlighted though that using this test, due to the typical independence violation of using the same test set as well as its high false positive rate, it is not recommended to use this test in practice.","title":"Example 1 - Difference of Proportions"},{"location":"user_guide/evaluate/proportion_difference/#api","text":"proportion_difference(proportion_1, proportion_2, n_1, n_2=None) Computes the test statistic and p-value for a difference of proportions test. Parameters proportion_1 : float The first proportion proportion_2 : float The second proportion n_1 : int The sample size of the first test sample n_2 : int or None (default=None) The sample size of the second test sample. If None , n_1 = n_2 . Returns z, p : float or None, float Returns the z-score and the p-value Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/proportion_difference/","title":"API"},{"location":"user_guide/evaluate/scoring/","text":"scoring: computing various performance metrics A function for computing various different performance metrics. from mlxtend.evaluate import scoring Overview Confusion Matrix The confusion matrix (or error matrix ) is one way to summarize the performance of a classifier for binary classification tasks. This square matrix consists of columns and rows that list the number of instances as absolute or relative \"actual class\" vs. \"predicted class\" ratios. Let P be the label of class 1 and N be the label of a second class or the label of all classes that are not class 1 in a multi-class setting. Error and Accuracy Both the prediction error (ERR) and accuracy (ACC) provide general information about how many samples are misclassified. The error can be understood as the sum of all false predictions divided by the number of total predications, and the the accuracy is calculated as the sum of correct predictions divided by the total number of predictions, respectively. ERR = \\frac{FP + FN}{FP+ FN + TP + TN} = 1-ACC ACC = \\frac{TP + TN}{FP+ FN + TP + TN} = 1-ERR True and False Positive Rates The True Positive Rate (TPR) and False Positive Rate (FPR) are performance metrics that are especially useful for imbalanced class problems. In spam classification , for example, we are of course primarily interested in the detection and filtering out of spam . However, it is also important to decrease the number of messages that were incorrectly classified as spam ( False Positives ): A situation where a person misses an important message is considered as \"worse\" than a situation where a person ends up with a few spam messages in his e-mail inbox. In contrast to the FPR , the True Positive Rate provides useful information about the fraction of positive (or relevant ) samples that were correctly identified out of the total pool of Positives . FPR = \\frac{FP}{N} = \\frac{FP}{FP + TN} TPR = \\frac{TP}{P} = \\frac{TP}{FN + TP} Precision, Recall, and the F1-Score Precision (PRE) and Recall (REC) are metrics that are more commonly used in Information Technology and related to the False and True Prositive Rates . In fact, Recall is synonymous to the True Positive Rate and also sometimes called Sensitivity . The F _1 -Score can be understood as a combination of both Precision and Recall . PRE = \\frac{TP}{TP + FP} REC = TPR = \\frac{TP}{P} = \\frac{TP}{FN + TP} F_1 = 2 \\cdot \\frac{PRE \\cdot REC}{PRE + REC} Sensitivity and Specificity Sensitivity (SEN) is synonymous to Recall and the True Positive Rate whereas Specificity (SPC) is synonymous to the True Negative Rate -- Sensitivity measures the recovery rate of the Positives and complimentary, the Specificity measures the recovery rate of the Negatives . SEN = TPR = REC = \\frac{TP}{P} = \\frac{TP}{FN + TP} SPC = TNR =\\frac{TN}{N} = \\frac{TN}{FP + TN} Matthews Correlation Coefficient Matthews correlation coefficient (MCC) was first formulated by Brian W. Matthews [3] in 1975 to assess the performance of protein secondary structure predictions. The MCC can be understood as a specific case of a linear correlation coefficient ( Pearson's R ) for a binary classification setting and is considered as especially useful in unbalanced class settings. The previous metrics take values in the range between 0 (worst) and 1 (best), whereas the MCC is bounded between the range 1 (perfect correlation between ground truth and predicted outcome) and -1 (inverse or negative correlation) -- a value of 0 denotes a random prediction. MCC = \\frac{ TP \\times TN - FP \\times FN } {\\sqrt{ (TP + FP) ( TP + FN ) ( TN + FP ) ( TN + FN ) } } Average Per-Class Accuracy The \"overall\" accuracy is defined as the number of correct predictions ( true positives TP and true negatives TN) over all samples n : ACC = \\frac{TP + TN}{n} in a binary class setting: In a multi-class setting, we can generalize the computation of the accuracy as the fraction of all true predictions (the diagonal) over all samples n. ACC = \\frac{T}{n} Considering a multi-class problem with 3 classes (C0, C1, C2) let's assume our model made the following predictions: We compute the accuracy as: ACC = \\frac{3 + 50 + 18}{90} \\approx 0.79 Now, in order to compute the average per-class accuracy , we compute the binary accuracy for each class label separately; i.e., if class 1 is the positive class, class 0 and 2 are both considered the negative class. APC\\;ACC = \\frac{83/90 + 71/90 + 78/90}{3} \\approx 0.86 References [1] S. Raschka. An overview of general performance metrics of binary classifier systems . Computing Research Repository (CoRR), abs/1410.5330, 2014. [2] Cyril Goutte and Eric Gaussier. A probabilistic interpretation of precision, recall and f-score, with implication for evaluation . In Advances in Information Retrieval, pages 345\u2013359. Springer, 2005. [3] Brian W Matthews. Comparison of the predicted and observed secondary structure of T4 phage lysozyme . Biochimica et Biophysica Acta (BBA)- Protein Structure, 405(2):442\u2013451, 1975. Example 1 - Classification Error from mlxtend.evaluate import scoring y_targ = [1, 1, 1, 0, 0, 2, 0, 3] y_pred = [1, 0, 1, 0, 0, 2, 1, 3] res = scoring(y_target=y_targ, y_predicted=y_pred, metric='error') print('Error: %s%%' % (res * 100)) Error: 25.0% API scoring(y_target, y_predicted, metric='error', positive_label=1, unique_labels='auto') Compute a scoring metric for supervised learning. Parameters y_target : array-like, shape=[n_values] True class labels or target values. y_predicted : array-like, shape=[n_values] Predicted class labels or target values. metric : str (default: 'error') Performance metric: 'accuracy': (TP + TN)/(FP + FN + TP + TN) = 1-ERR 'average per-class accuracy': Average per-class accuracy 'average per-class error': Average per-class error 'error': (TP + TN)/(FP+ FN + TP + TN) = 1-ACC 'false_positive_rate': FP/N = FP/(FP + TN) 'true_positive_rate': TP/P = TP/(FN + TP) 'true_negative_rate': TN/N = TN/(FP + TN) 'precision': TP/(TP + FP) 'recall': equal to 'true_positive_rate' 'sensitivity': equal to 'true_positive_rate' or 'recall' 'specificity': equal to 'true_negative_rate' 'f1': 2 * (PRE * REC)/(PRE + REC) 'matthews_corr_coef': (TP TN - FP FN) / (sqrt{(TP + FP)( TP + FN )( TN + FP )( TN + FN )}) Where: [TP: True positives, TN = True negatives, TN: True negatives, FN = False negatives] positive_label : int (default: 1) Label of the positive class for binary classification metrics. unique_labels : str or array-like (default: 'auto') If 'auto', deduces the unique class labels from y_target Returns score : float Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/scoring/","title":"scoring: computing various performance metrics"},{"location":"user_guide/evaluate/scoring/#scoring-computing-various-performance-metrics","text":"A function for computing various different performance metrics. from mlxtend.evaluate import scoring","title":"scoring: computing various performance metrics"},{"location":"user_guide/evaluate/scoring/#overview","text":"","title":"Overview"},{"location":"user_guide/evaluate/scoring/#confusion-matrix","text":"The confusion matrix (or error matrix ) is one way to summarize the performance of a classifier for binary classification tasks. This square matrix consists of columns and rows that list the number of instances as absolute or relative \"actual class\" vs. \"predicted class\" ratios. Let P be the label of class 1 and N be the label of a second class or the label of all classes that are not class 1 in a multi-class setting.","title":"Confusion Matrix"},{"location":"user_guide/evaluate/scoring/#error-and-accuracy","text":"Both the prediction error (ERR) and accuracy (ACC) provide general information about how many samples are misclassified. The error can be understood as the sum of all false predictions divided by the number of total predications, and the the accuracy is calculated as the sum of correct predictions divided by the total number of predictions, respectively. ERR = \\frac{FP + FN}{FP+ FN + TP + TN} = 1-ACC ACC = \\frac{TP + TN}{FP+ FN + TP + TN} = 1-ERR","title":"Error and Accuracy"},{"location":"user_guide/evaluate/scoring/#true-and-false-positive-rates","text":"The True Positive Rate (TPR) and False Positive Rate (FPR) are performance metrics that are especially useful for imbalanced class problems. In spam classification , for example, we are of course primarily interested in the detection and filtering out of spam . However, it is also important to decrease the number of messages that were incorrectly classified as spam ( False Positives ): A situation where a person misses an important message is considered as \"worse\" than a situation where a person ends up with a few spam messages in his e-mail inbox. In contrast to the FPR , the True Positive Rate provides useful information about the fraction of positive (or relevant ) samples that were correctly identified out of the total pool of Positives . FPR = \\frac{FP}{N} = \\frac{FP}{FP + TN} TPR = \\frac{TP}{P} = \\frac{TP}{FN + TP}","title":"True and False Positive Rates"},{"location":"user_guide/evaluate/scoring/#precision-recall-and-the-f1-score","text":"Precision (PRE) and Recall (REC) are metrics that are more commonly used in Information Technology and related to the False and True Prositive Rates . In fact, Recall is synonymous to the True Positive Rate and also sometimes called Sensitivity . The F _1 -Score can be understood as a combination of both Precision and Recall . PRE = \\frac{TP}{TP + FP} REC = TPR = \\frac{TP}{P} = \\frac{TP}{FN + TP} F_1 = 2 \\cdot \\frac{PRE \\cdot REC}{PRE + REC}","title":"Precision, Recall, and the F1-Score"},{"location":"user_guide/evaluate/scoring/#sensitivity-and-specificity","text":"Sensitivity (SEN) is synonymous to Recall and the True Positive Rate whereas Specificity (SPC) is synonymous to the True Negative Rate -- Sensitivity measures the recovery rate of the Positives and complimentary, the Specificity measures the recovery rate of the Negatives . SEN = TPR = REC = \\frac{TP}{P} = \\frac{TP}{FN + TP} SPC = TNR =\\frac{TN}{N} = \\frac{TN}{FP + TN}","title":"Sensitivity and Specificity"},{"location":"user_guide/evaluate/scoring/#matthews-correlation-coefficient","text":"Matthews correlation coefficient (MCC) was first formulated by Brian W. Matthews [3] in 1975 to assess the performance of protein secondary structure predictions. The MCC can be understood as a specific case of a linear correlation coefficient ( Pearson's R ) for a binary classification setting and is considered as especially useful in unbalanced class settings. The previous metrics take values in the range between 0 (worst) and 1 (best), whereas the MCC is bounded between the range 1 (perfect correlation between ground truth and predicted outcome) and -1 (inverse or negative correlation) -- a value of 0 denotes a random prediction. MCC = \\frac{ TP \\times TN - FP \\times FN } {\\sqrt{ (TP + FP) ( TP + FN ) ( TN + FP ) ( TN + FN ) } }","title":"Matthews Correlation Coefficient"},{"location":"user_guide/evaluate/scoring/#average-per-class-accuracy","text":"The \"overall\" accuracy is defined as the number of correct predictions ( true positives TP and true negatives TN) over all samples n : ACC = \\frac{TP + TN}{n} in a binary class setting: In a multi-class setting, we can generalize the computation of the accuracy as the fraction of all true predictions (the diagonal) over all samples n. ACC = \\frac{T}{n} Considering a multi-class problem with 3 classes (C0, C1, C2) let's assume our model made the following predictions: We compute the accuracy as: ACC = \\frac{3 + 50 + 18}{90} \\approx 0.79 Now, in order to compute the average per-class accuracy , we compute the binary accuracy for each class label separately; i.e., if class 1 is the positive class, class 0 and 2 are both considered the negative class. APC\\;ACC = \\frac{83/90 + 71/90 + 78/90}{3} \\approx 0.86","title":"Average Per-Class Accuracy"},{"location":"user_guide/evaluate/scoring/#references","text":"[1] S. Raschka. An overview of general performance metrics of binary classifier systems . Computing Research Repository (CoRR), abs/1410.5330, 2014. [2] Cyril Goutte and Eric Gaussier. A probabilistic interpretation of precision, recall and f-score, with implication for evaluation . In Advances in Information Retrieval, pages 345\u2013359. Springer, 2005. [3] Brian W Matthews. Comparison of the predicted and observed secondary structure of T4 phage lysozyme . Biochimica et Biophysica Acta (BBA)- Protein Structure, 405(2):442\u2013451, 1975.","title":"References"},{"location":"user_guide/evaluate/scoring/#example-1-classification-error","text":"from mlxtend.evaluate import scoring y_targ = [1, 1, 1, 0, 0, 2, 0, 3] y_pred = [1, 0, 1, 0, 0, 2, 1, 3] res = scoring(y_target=y_targ, y_predicted=y_pred, metric='error') print('Error: %s%%' % (res * 100)) Error: 25.0%","title":"Example 1 - Classification Error"},{"location":"user_guide/evaluate/scoring/#api","text":"scoring(y_target, y_predicted, metric='error', positive_label=1, unique_labels='auto') Compute a scoring metric for supervised learning. Parameters y_target : array-like, shape=[n_values] True class labels or target values. y_predicted : array-like, shape=[n_values] Predicted class labels or target values. metric : str (default: 'error') Performance metric: 'accuracy': (TP + TN)/(FP + FN + TP + TN) = 1-ERR 'average per-class accuracy': Average per-class accuracy 'average per-class error': Average per-class error 'error': (TP + TN)/(FP+ FN + TP + TN) = 1-ACC 'false_positive_rate': FP/N = FP/(FP + TN) 'true_positive_rate': TP/P = TP/(FN + TP) 'true_negative_rate': TN/N = TN/(FP + TN) 'precision': TP/(TP + FP) 'recall': equal to 'true_positive_rate' 'sensitivity': equal to 'true_positive_rate' or 'recall' 'specificity': equal to 'true_negative_rate' 'f1': 2 * (PRE * REC)/(PRE + REC) 'matthews_corr_coef': (TP TN - FP FN) / (sqrt{(TP + FP)( TP + FN )( TN + FP )( TN + FN )}) Where: [TP: True positives, TN = True negatives, TN: True negatives, FN = False negatives] positive_label : int (default: 1) Label of the positive class for binary classification metrics. unique_labels : str or array-like (default: 'auto') If 'auto', deduces the unique class labels from y_target Returns score : float Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/scoring/","title":"API"},{"location":"user_guide/feature_extraction/LinearDiscriminantAnalysis/","text":"LinearDiscriminantAnalysis: Linear discriminant analysis for dimensionality reduction Implementation of Linear Discriminant Analysis for dimensionality reduction from mlxtend.feature_extraction import LinearDiscriminantAnalysis Overview Linear Discriminant Analysis (LDA) is most commonly used as dimensionality reduction technique in the pre-processing step for pattern-classification and machine learning applications. The goal is to project a dataset onto a lower-dimensional space with good class-separability in order avoid overfitting (\"curse of dimensionality\") and also reduce computational costs. Ronald A. Fisher formulated the Linear Discriminant in 1936 ( The Use of Multiple Measurements in Taxonomic Problems ), and it also has some practical uses as classifier. The original Linear discriminant was described for a 2-class problem, and it was then later generalized as \"multi-class Linear Discriminant Analysis\" or \"Multiple Discriminant Analysis\" by C. R. Rao in 1948 ( The utilization of multiple measurements in problems of biological classification ) The general LDA approach is very similar to a Principal Component Analysis, but in addition to finding the component axes that maximize the variance of our data (PCA), we are additionally interested in the axes that maximize the separation between multiple classes (LDA). So, in a nutshell, often the goal of an LDA is to project a feature space (a dataset n-dimensional samples) onto a smaller subspace k (where k \\leq n-1 ) while maintaining the class-discriminatory information. In general, dimensionality reduction does not only help reducing computational costs for a given classification task, but it can also be helpful to avoid overfitting by minimizing the error in parameter estimation (\"curse of dimensionality\"). Summarizing the LDA approach in 5 steps Listed below are the 5 general steps for performing a linear discriminant analysis. Compute the d -dimensional mean vectors for the different classes from the dataset. Compute the scatter matrices (in-between-class and within-class scatter matrix). Compute the eigenvectors ( \\mathbf{e_1}, \\; \\mathbf{e_2}, \\; ..., \\; \\mathbf{e_d} ) and corresponding eigenvalues ( \\mathbf{\\lambda_1}, \\; \\mathbf{\\lambda_2}, \\; ..., \\; \\mathbf{\\lambda_d} ) for the scatter matrices. Sort the eigenvectors by decreasing eigenvalues and choose k eigenvectors with the largest eigenvalues to form a k \\times d dimensional matrix \\mathbf{W} (where every column represents an eigenvector). Use this k \\times d eigenvector matrix to transform the samples onto the new subspace. This can be summarized by the mathematical equation: \\mathbf{Y} = \\mathbf{X} \\times \\mathbf{W} (where \\mathbf{X} is a n \\times d -dimensional matrix representing the n samples, and \\mathbf{y} are the transformed n \\times k -dimensional samples in the new subspace). References Fisher, Ronald A. \" The use of multiple measurements in taxonomic problems. \" Annals of eugenics 7.2 (1936): 179-188. Rao, C. Radhakrishna. \" The utilization of multiple measurements in problems of biological classification. \" Journal of the Royal Statistical Society. Series B (Methodological) 10.2 (1948): 159-203. Example 1 - LDA on Iris from mlxtend.data import iris_data from mlxtend.preprocessing import standardize from mlxtend.feature_extraction import LinearDiscriminantAnalysis X, y = iris_data() X = standardize(X) lda = LinearDiscriminantAnalysis(n_discriminants=2) lda.fit(X, y) X_lda = lda.transform(X) import matplotlib.pyplot as plt with plt.style.context('seaborn-whitegrid'): plt.figure(figsize=(6, 4)) for lab, col in zip((0, 1, 2), ('blue', 'red', 'green')): plt.scatter(X_lda[y == lab, 0], X_lda[y == lab, 1], label=lab, c=col) plt.xlabel('Linear Discriminant 1') plt.ylabel('Linear Discriminant 2') plt.legend(loc='lower right') plt.tight_layout() plt.show() Example 2 - Plotting the Between-Class Variance Explained Ratio from mlxtend.data import iris_data from mlxtend.preprocessing import standardize from mlxtend.feature_extraction import LinearDiscriminantAnalysis X, y = iris_data() X = standardize(X) lda = LinearDiscriminantAnalysis(n_discriminants=None) lda.fit(X, y) X_lda = lda.transform(X) import numpy as np tot = sum(lda.e_vals_) var_exp = [(i / tot)*100 for i in sorted(lda.e_vals_, reverse=True)] cum_var_exp = np.cumsum(var_exp) with plt.style.context('seaborn-whitegrid'): fig, ax = plt.subplots(figsize=(6, 4)) plt.bar(range(4), var_exp, alpha=0.5, align='center', label='individual explained variance') plt.step(range(4), cum_var_exp, where='mid', label='cumulative explained variance') plt.ylabel('Explained variance ratio') plt.xlabel('Principal components') plt.xticks(range(4)) ax.set_xticklabels(np.arange(1, X.shape[1] + 1)) plt.legend(loc='best') plt.tight_layout() API LinearDiscriminantAnalysis(n_discriminants=None) Linear Discriminant Analysis Class Parameters n_discriminants : int (default: None) The number of discrimants for transformation. Keeps the original dimensions of the dataset if None . Attributes w_ : array-like, shape=[n_features, n_discriminants] Projection matrix e_vals_ : array-like, shape=[n_features] Eigenvalues in sorted order. e_vecs_ : array-like, shape=[n_features] Eigenvectors in sorted order. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/feature_extraction/LinearDiscriminantAnalysis/ Methods fit(X, y, n_classes=None) Fit the LDA model with X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values. n_classes : int (default: None) A positive integer to declare the number of class labels if not all class labels are present in a partial training set. Gets the number of class labels automatically if None. Returns self : object transform(X) Apply the linear transformation on X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns X_projected : np.ndarray, shape = [n_samples, n_discriminants] Projected training vectors.","title":"LinearDiscriminantAnalysis: Linear discriminant analysis for dimensionality reduction"},{"location":"user_guide/feature_extraction/LinearDiscriminantAnalysis/#lineardiscriminantanalysis-linear-discriminant-analysis-for-dimensionality-reduction","text":"Implementation of Linear Discriminant Analysis for dimensionality reduction from mlxtend.feature_extraction import LinearDiscriminantAnalysis","title":"LinearDiscriminantAnalysis: Linear discriminant analysis for dimensionality reduction"},{"location":"user_guide/feature_extraction/LinearDiscriminantAnalysis/#overview","text":"Linear Discriminant Analysis (LDA) is most commonly used as dimensionality reduction technique in the pre-processing step for pattern-classification and machine learning applications. The goal is to project a dataset onto a lower-dimensional space with good class-separability in order avoid overfitting (\"curse of dimensionality\") and also reduce computational costs. Ronald A. Fisher formulated the Linear Discriminant in 1936 ( The Use of Multiple Measurements in Taxonomic Problems ), and it also has some practical uses as classifier. The original Linear discriminant was described for a 2-class problem, and it was then later generalized as \"multi-class Linear Discriminant Analysis\" or \"Multiple Discriminant Analysis\" by C. R. Rao in 1948 ( The utilization of multiple measurements in problems of biological classification ) The general LDA approach is very similar to a Principal Component Analysis, but in addition to finding the component axes that maximize the variance of our data (PCA), we are additionally interested in the axes that maximize the separation between multiple classes (LDA). So, in a nutshell, often the goal of an LDA is to project a feature space (a dataset n-dimensional samples) onto a smaller subspace k (where k \\leq n-1 ) while maintaining the class-discriminatory information. In general, dimensionality reduction does not only help reducing computational costs for a given classification task, but it can also be helpful to avoid overfitting by minimizing the error in parameter estimation (\"curse of dimensionality\").","title":"Overview"},{"location":"user_guide/feature_extraction/LinearDiscriminantAnalysis/#summarizing-the-lda-approach-in-5-steps","text":"Listed below are the 5 general steps for performing a linear discriminant analysis. Compute the d -dimensional mean vectors for the different classes from the dataset. Compute the scatter matrices (in-between-class and within-class scatter matrix). Compute the eigenvectors ( \\mathbf{e_1}, \\; \\mathbf{e_2}, \\; ..., \\; \\mathbf{e_d} ) and corresponding eigenvalues ( \\mathbf{\\lambda_1}, \\; \\mathbf{\\lambda_2}, \\; ..., \\; \\mathbf{\\lambda_d} ) for the scatter matrices. Sort the eigenvectors by decreasing eigenvalues and choose k eigenvectors with the largest eigenvalues to form a k \\times d dimensional matrix \\mathbf{W} (where every column represents an eigenvector). Use this k \\times d eigenvector matrix to transform the samples onto the new subspace. This can be summarized by the mathematical equation: \\mathbf{Y} = \\mathbf{X} \\times \\mathbf{W} (where \\mathbf{X} is a n \\times d -dimensional matrix representing the n samples, and \\mathbf{y} are the transformed n \\times k -dimensional samples in the new subspace).","title":"Summarizing the LDA approach in 5 steps"},{"location":"user_guide/feature_extraction/LinearDiscriminantAnalysis/#references","text":"Fisher, Ronald A. \" The use of multiple measurements in taxonomic problems. \" Annals of eugenics 7.2 (1936): 179-188. Rao, C. Radhakrishna. \" The utilization of multiple measurements in problems of biological classification. \" Journal of the Royal Statistical Society. Series B (Methodological) 10.2 (1948): 159-203.","title":"References"},{"location":"user_guide/feature_extraction/LinearDiscriminantAnalysis/#example-1-lda-on-iris","text":"from mlxtend.data import iris_data from mlxtend.preprocessing import standardize from mlxtend.feature_extraction import LinearDiscriminantAnalysis X, y = iris_data() X = standardize(X) lda = LinearDiscriminantAnalysis(n_discriminants=2) lda.fit(X, y) X_lda = lda.transform(X) import matplotlib.pyplot as plt with plt.style.context('seaborn-whitegrid'): plt.figure(figsize=(6, 4)) for lab, col in zip((0, 1, 2), ('blue', 'red', 'green')): plt.scatter(X_lda[y == lab, 0], X_lda[y == lab, 1], label=lab, c=col) plt.xlabel('Linear Discriminant 1') plt.ylabel('Linear Discriminant 2') plt.legend(loc='lower right') plt.tight_layout() plt.show()","title":"Example 1 - LDA on Iris"},{"location":"user_guide/feature_extraction/LinearDiscriminantAnalysis/#example-2-plotting-the-between-class-variance-explained-ratio","text":"from mlxtend.data import iris_data from mlxtend.preprocessing import standardize from mlxtend.feature_extraction import LinearDiscriminantAnalysis X, y = iris_data() X = standardize(X) lda = LinearDiscriminantAnalysis(n_discriminants=None) lda.fit(X, y) X_lda = lda.transform(X) import numpy as np tot = sum(lda.e_vals_) var_exp = [(i / tot)*100 for i in sorted(lda.e_vals_, reverse=True)] cum_var_exp = np.cumsum(var_exp) with plt.style.context('seaborn-whitegrid'): fig, ax = plt.subplots(figsize=(6, 4)) plt.bar(range(4), var_exp, alpha=0.5, align='center', label='individual explained variance') plt.step(range(4), cum_var_exp, where='mid', label='cumulative explained variance') plt.ylabel('Explained variance ratio') plt.xlabel('Principal components') plt.xticks(range(4)) ax.set_xticklabels(np.arange(1, X.shape[1] + 1)) plt.legend(loc='best') plt.tight_layout()","title":"Example 2 - Plotting the Between-Class Variance Explained Ratio"},{"location":"user_guide/feature_extraction/LinearDiscriminantAnalysis/#api","text":"LinearDiscriminantAnalysis(n_discriminants=None) Linear Discriminant Analysis Class Parameters n_discriminants : int (default: None) The number of discrimants for transformation. Keeps the original dimensions of the dataset if None . Attributes w_ : array-like, shape=[n_features, n_discriminants] Projection matrix e_vals_ : array-like, shape=[n_features] Eigenvalues in sorted order. e_vecs_ : array-like, shape=[n_features] Eigenvectors in sorted order. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/feature_extraction/LinearDiscriminantAnalysis/","title":"API"},{"location":"user_guide/feature_extraction/LinearDiscriminantAnalysis/#methods","text":"fit(X, y, n_classes=None) Fit the LDA model with X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values. n_classes : int (default: None) A positive integer to declare the number of class labels if not all class labels are present in a partial training set. Gets the number of class labels automatically if None. Returns self : object transform(X) Apply the linear transformation on X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns X_projected : np.ndarray, shape = [n_samples, n_discriminants] Projected training vectors.","title":"Methods"},{"location":"user_guide/feature_extraction/PrincipalComponentAnalysis/","text":"PrincipalComponentAnalysis: Principal component analysis (PCA) for dimensionality reduction Implementation of Principal Component Analysis for dimensionality reduction from mlxtend.feature_extraction import PrincipalComponentAnalysis Overview The sheer size of data in the modern age is not only a challenge for computer hardware but also a main bottleneck for the performance of many machine learning algorithms. The main goal of a PCA analysis is to identify patterns in data; PCA aims to detect the correlation between variables. If a strong correlation between variables exists, the attempt to reduce the dimensionality only makes sense. In a nutshell, this is what PCA is all about: Finding the directions of maximum variance in high-dimensional data and project it onto a smaller dimensional subspace while retaining most of the information. PCA and Dimensionality Reduction Often, the desired goal is to reduce the dimensions of a d -dimensional dataset by projecting it onto a (k) -dimensional subspace (where k\\;<\\;d ) in order to increase the computational efficiency while retaining most of the information. An important question is \"what is the size of k that represents the data 'well'?\" Later, we will compute eigenvectors (the principal components) of a dataset and collect them in a projection matrix. Each of those eigenvectors is associated with an eigenvalue which can be interpreted as the \"length\" or \"magnitude\" of the corresponding eigenvector. If some eigenvalues have a significantly larger magnitude than others that the reduction of the dataset via PCA onto a smaller dimensional subspace by dropping the \"less informative\" eigenpairs is reasonable. A Summary of the PCA Approach Standardize the data. Obtain the Eigenvectors and Eigenvalues from the covariance matrix or correlation matrix, or perform Singular Vector Decomposition. Sort eigenvalues in descending order and choose the k eigenvectors that correspond to the k largest eigenvalues where k is the number of dimensions of the new feature subspace ( k \\le d ). Construct the projection matrix \\mathbf{W} from the selected k eigenvectors. Transform the original dataset \\mathbf{X} via \\mathbf{W} to obtain a k -dimensional feature subspace \\mathbf{Y} . References Pearson, Karl. \"LIII. On lines and planes of closest fit to systems of points in space. \" The London, Edinburgh, and Dublin Philosophical Magazine and Journal of Science 2.11 (1901): 559-572. Example 1 - PCA on Iris from mlxtend.data import iris_data from mlxtend.preprocessing import standardize from mlxtend.feature_extraction import PrincipalComponentAnalysis X, y = iris_data() X = standardize(X) pca = PrincipalComponentAnalysis(n_components=2) pca.fit(X) X_pca = pca.transform(X) import matplotlib.pyplot as plt with plt.style.context('seaborn-whitegrid'): plt.figure(figsize=(6, 4)) for lab, col in zip((0, 1, 2), ('blue', 'red', 'green')): plt.scatter(X_pca[y==lab, 0], X_pca[y==lab, 1], label=lab, c=col) plt.xlabel('Principal Component 1') plt.ylabel('Principal Component 2') plt.legend(loc='lower center') plt.tight_layout() plt.show() Example 2 - Plotting the Variance Explained Ratio from mlxtend.data import iris_data from mlxtend.preprocessing import standardize X, y = iris_data() X = standardize(X) pca = PrincipalComponentAnalysis(n_components=None) pca.fit(X) X_pca = pca.transform(X) pca.e_vals_ array([2.91081808, 0.92122093, 0.14735328, 0.02060771]) pca.e_vals_normalized_ array([0.72770452, 0.23030523, 0.03683832, 0.00515193]) import numpy as np tot = sum(pca.e_vals_) var_exp = [(i / tot)*100 for i in sorted(pca.e_vals_, reverse=True)] cum_var_exp = np.cumsum(pca.e_vals_normalized_*100) with plt.style.context('seaborn-whitegrid'): fig, ax = plt.subplots(figsize=(6, 4)) plt.bar(range(4), var_exp, alpha=0.5, align='center', label='individual explained variance') plt.step(range(4), cum_var_exp, where='mid', label='cumulative explained variance') plt.ylabel('Explained variance ratio') plt.xlabel('Principal components') plt.xticks(range(4)) ax.set_xticklabels(np.arange(1, X.shape[1] + 1)) plt.legend(loc='best') plt.tight_layout() Example 3 - PCA via SVD While the eigendecomposition of the covariance or correlation matrix may be more intuitiuve, most PCA implementations perform a Singular Vector Decomposition (SVD) to improve the computational efficiency. Another advantage of using SVD is that the results tend to be more numerically stable, since we can decompose the input matrix directly without the additional covariance-matrix step. from mlxtend.data import iris_data from mlxtend.preprocessing import standardize from mlxtend.feature_extraction import PrincipalComponentAnalysis X, y = iris_data() X = standardize(X) pca = PrincipalComponentAnalysis(n_components=2, solver='svd') pca.fit(X) X_pca = pca.transform(X) import matplotlib.pyplot as plt with plt.style.context('seaborn-whitegrid'): plt.figure(figsize=(6, 4)) for lab, col in zip((0, 1, 2), ('blue', 'red', 'green')): plt.scatter(X_pca[y==lab, 0], X_pca[y==lab, 1], label=lab, c=col) plt.xlabel('Principal Component 1') plt.ylabel('Principal Component 2') plt.legend(loc='lower center') plt.tight_layout() plt.show() If we compare this PCA projection to the previous plot in example 1, we notice that they are mirror images of each other. Note that this is not due to an error in any of those two implementations, but the reason for this difference is that, depending on the eigensolver, eigenvectors can have either negative or positive signs. For instance, if v is an eigenvector of a matrix \\Sigma , we have \\Sigma v = \\lambda v, where \\lambda is our eigenvalue then -v is also an eigenvector that has the same eigenvalue, since \\Sigma(-v) = -\\Sigma v = -\\lambda v = \\lambda(-v). Example 4 - Factor Loadings After evoking the fit method, the factor loadings are available via the loadings_ attribute. In simple terms, the loadings are the unstandardized values of the eigenvectors. Or in other words, we can interpret the loadings as the covariances (or correlation in case we standardized the input features) between the input features and the principal components (or eigenvectors), which have been scaled to unit length. By having the loadings scaled, they become comparable by magnitude and we can assess how much variance in a component is attributed to the input features (as the components are just a weighted linear combination of the input features). from mlxtend.data import iris_data from mlxtend.preprocessing import standardize from mlxtend.feature_extraction import PrincipalComponentAnalysis import matplotlib.pyplot as plt X, y = iris_data() X = standardize(X) pca = PrincipalComponentAnalysis(n_components=2, solver='eigen') pca.fit(X); xlabels = ['sepal length', 'sepal width', 'petal length', 'petal width'] fig, ax = plt.subplots(1, 2, figsize=(8, 3)) ax[0].bar(range(4), pca.loadings_[:, 0], align='center') ax[1].bar(range(4), pca.loadings_[:, 1], align='center') ax[0].set_ylabel('Factor loading onto PC1') ax[1].set_ylabel('Factor loading onto PC2') ax[0].set_xticks(range(4)) ax[1].set_xticks(range(4)) ax[0].set_xticklabels(xlabels, rotation=45) ax[1].set_xticklabels(xlabels, rotation=45) plt.ylim([-1, 1]) plt.tight_layout() For instance, we may say that most of the variance in the first component is attributed to the petal features (although the loading of sepal length on PC1 is also not much less in magnitude). In contrast, the remaining variance captured by PC2 is mostly due to the sepal width. Note that we know from Example 2 that PC1 explains most of the variance, and based on the information from the loading plots, we may say that petal features combined with sepal length may explain most of the spread in the data. Example 5 - Feature Extraction Pipeline from sklearn.pipeline import make_pipeline from sklearn.neighbors import KNeighborsClassifier from sklearn.preprocessing import StandardScaler from sklearn.model_selection import train_test_split from mlxtend.data import wine_data X, y = wine_data() X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=123, test_size=0.3, stratify=y) pipe_pca = make_pipeline(StandardScaler(), PrincipalComponentAnalysis(n_components=3), KNeighborsClassifier(n_neighbors=5)) pipe_pca.fit(X_train, y_train) print('Transf. training accyracy: %.2f%%' % (pipe_pca.score(X_train, y_train)*100)) print('Transf. test accyracy: %.2f%%' % (pipe_pca.score(X_test, y_test)*100)) Transf. training accyracy: 96.77% Transf. test accyracy: 96.30% Example 6 - Whitening Certain algorithms require the data to be whitened. This means that the features have unit variance and the off-diagonals are all zero (i.e., the features are uncorrelated). PCA already ensures that the features are uncorrelated, hence, we only need to apply a simple scaling to whiten the transformed data. For instance, for a given transformed feature X'_i , we divide it by the square-root of the corresponding eigenvalue \\lambda_i : X'_{\\text{whitened}} = \\frac{X'_i}{\\sqrt{\\lambda_i}}. The whitening via the PrincipalComponentAnalysis can be achieved by setting whitening=True during initialization. Let's demonstrate that with an example. from sklearn.preprocessing import StandardScaler from sklearn.model_selection import train_test_split from mlxtend.data import wine_data X, y = wine_data() X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=123, test_size=0.3, stratify=y) Regular PCA sc = StandardScaler() pca1 = PrincipalComponentAnalysis(n_components=2) X_train_scaled = sc.fit_transform(X_train) X_train_transf = pca1.fit(X_train_scaled).transform(X_train_scaled) with plt.style.context('seaborn-whitegrid'): plt.figure(figsize=(6, 4)) for lab, col in zip((0, 1, 2), ('blue', 'red', 'green')): plt.scatter(X_train_transf[y_train==lab, 0], X_train_transf[y_train==lab, 1], label=lab, c=col) plt.xlabel('Principal Component 1') plt.ylabel('Principal Component 2') plt.legend(loc='lower center') plt.tight_layout() plt.show() np.set_printoptions(precision=1, suppress=True) print('Covariance matrix:\\n') np.cov(X_train_transf.T) Covariance matrix: array([[4.9, 0. ], [0. , 2.5]]) As we can see, the features are uncorrelated after transformation but don't have unit variance. PCA with Whitening sc = StandardScaler() pca1 = PrincipalComponentAnalysis(n_components=2, whitening=True) X_train_scaled = sc.fit_transform(X_train) X_train_transf = pca1.fit(X_train_scaled).transform(X_train_scaled) with plt.style.context('seaborn-whitegrid'): plt.figure(figsize=(6, 4)) for lab, col in zip((0, 1, 2), ('blue', 'red', 'green')): plt.scatter(X_train_transf[y_train==lab, 0], X_train_transf[y_train==lab, 1], label=lab, c=col) plt.xlabel('Principal Component 1') plt.ylabel('Principal Component 2') plt.legend(loc='lower center') plt.tight_layout() plt.show() np.set_printoptions(precision=1, suppress=True) print('Covariance matrix:\\n') np.cov(X_train_transf.T) Covariance matrix: array([[1., 0.], [0., 1.]]) As we can see above, the whitening achieves that all features now have unit variance. I.e., the covariance matrix of the transformed features becomes the identity matrix. API PrincipalComponentAnalysis(n_components=None, solver='svd', whitening=False) Principal Component Analysis Class Parameters n_components : int (default: None) The number of principal components for transformation. Keeps the original dimensions of the dataset if None . solver : str (default: 'svd') Method for performing the matrix decomposition. {'eigen', 'svd'} whitening : bool (default: False) Performs whitening such that the covariance matrix of the transformed data will be the identity matrix. Attributes w_ : array-like, shape=[n_features, n_components] Projection matrix e_vals_ : array-like, shape=[n_features] Eigenvalues in sorted order. e_vecs_ : array-like, shape=[n_features] Eigenvectors in sorted order. e_vals_normalized_ : array-like, shape=[n_features] Normalized eigen values such that they sum up to 1. This is equal to what's often referred to as \"explained variance ratios.\" loadings_ : array_like, shape=[n_features, n_features] The factor loadings of the original variables onto the principal components. The columns are the principal components, and the rows are the features loadings. For instance, the first column contains the loadings onto the first principal component. Note that the signs may be flipped depending on whether you use the 'eigen' or 'svd' solver; this does not affect the interpretation of the loadings though. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/feature_extraction/PrincipalComponentAnalysis/ Methods fit(X, y=None) Learn model from training data. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns self : object get_params(deep=True) Get parameters for this estimator. Parameters deep : boolean, optional If True, will return the parameters for this estimator and contained subobjects that are estimators. Returns params : mapping of string to any Parameter names mapped to their values.' adapted from https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/base.py Author: Gael Varoquaux gael.varoquaux@normalesup.org License: BSD 3 clause set_params( params) Set the parameters of this estimator. The method works on simple estimators as well as on nested objects (such as pipelines). The latter have parameters of the form <component>__<parameter> so that it's possible to update each component of a nested object. Returns self adapted from https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/base.py Author: Gael Varoquaux gael.varoquaux@normalesup.org License: BSD 3 clause transform(X) Apply the linear transformation on X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns X_projected : np.ndarray, shape = [n_samples, n_components] Projected training vectors.","title":"PrincipalComponentAnalysis: Principal component analysis (PCA) for dimensionality reduction"},{"location":"user_guide/feature_extraction/PrincipalComponentAnalysis/#principalcomponentanalysis-principal-component-analysis-pca-for-dimensionality-reduction","text":"Implementation of Principal Component Analysis for dimensionality reduction from mlxtend.feature_extraction import PrincipalComponentAnalysis","title":"PrincipalComponentAnalysis: Principal component analysis (PCA) for dimensionality reduction"},{"location":"user_guide/feature_extraction/PrincipalComponentAnalysis/#overview","text":"The sheer size of data in the modern age is not only a challenge for computer hardware but also a main bottleneck for the performance of many machine learning algorithms. The main goal of a PCA analysis is to identify patterns in data; PCA aims to detect the correlation between variables. If a strong correlation between variables exists, the attempt to reduce the dimensionality only makes sense. In a nutshell, this is what PCA is all about: Finding the directions of maximum variance in high-dimensional data and project it onto a smaller dimensional subspace while retaining most of the information.","title":"Overview"},{"location":"user_guide/feature_extraction/PrincipalComponentAnalysis/#pca-and-dimensionality-reduction","text":"Often, the desired goal is to reduce the dimensions of a d -dimensional dataset by projecting it onto a (k) -dimensional subspace (where k\\;<\\;d ) in order to increase the computational efficiency while retaining most of the information. An important question is \"what is the size of k that represents the data 'well'?\" Later, we will compute eigenvectors (the principal components) of a dataset and collect them in a projection matrix. Each of those eigenvectors is associated with an eigenvalue which can be interpreted as the \"length\" or \"magnitude\" of the corresponding eigenvector. If some eigenvalues have a significantly larger magnitude than others that the reduction of the dataset via PCA onto a smaller dimensional subspace by dropping the \"less informative\" eigenpairs is reasonable.","title":"PCA and Dimensionality Reduction"},{"location":"user_guide/feature_extraction/PrincipalComponentAnalysis/#a-summary-of-the-pca-approach","text":"Standardize the data. Obtain the Eigenvectors and Eigenvalues from the covariance matrix or correlation matrix, or perform Singular Vector Decomposition. Sort eigenvalues in descending order and choose the k eigenvectors that correspond to the k largest eigenvalues where k is the number of dimensions of the new feature subspace ( k \\le d ). Construct the projection matrix \\mathbf{W} from the selected k eigenvectors. Transform the original dataset \\mathbf{X} via \\mathbf{W} to obtain a k -dimensional feature subspace \\mathbf{Y} .","title":"A Summary of the PCA Approach"},{"location":"user_guide/feature_extraction/PrincipalComponentAnalysis/#references","text":"Pearson, Karl. \"LIII. On lines and planes of closest fit to systems of points in space. \" The London, Edinburgh, and Dublin Philosophical Magazine and Journal of Science 2.11 (1901): 559-572.","title":"References"},{"location":"user_guide/feature_extraction/PrincipalComponentAnalysis/#example-1-pca-on-iris","text":"from mlxtend.data import iris_data from mlxtend.preprocessing import standardize from mlxtend.feature_extraction import PrincipalComponentAnalysis X, y = iris_data() X = standardize(X) pca = PrincipalComponentAnalysis(n_components=2) pca.fit(X) X_pca = pca.transform(X) import matplotlib.pyplot as plt with plt.style.context('seaborn-whitegrid'): plt.figure(figsize=(6, 4)) for lab, col in zip((0, 1, 2), ('blue', 'red', 'green')): plt.scatter(X_pca[y==lab, 0], X_pca[y==lab, 1], label=lab, c=col) plt.xlabel('Principal Component 1') plt.ylabel('Principal Component 2') plt.legend(loc='lower center') plt.tight_layout() plt.show()","title":"Example 1 - PCA on Iris"},{"location":"user_guide/feature_extraction/PrincipalComponentAnalysis/#example-2-plotting-the-variance-explained-ratio","text":"from mlxtend.data import iris_data from mlxtend.preprocessing import standardize X, y = iris_data() X = standardize(X) pca = PrincipalComponentAnalysis(n_components=None) pca.fit(X) X_pca = pca.transform(X) pca.e_vals_ array([2.91081808, 0.92122093, 0.14735328, 0.02060771]) pca.e_vals_normalized_ array([0.72770452, 0.23030523, 0.03683832, 0.00515193]) import numpy as np tot = sum(pca.e_vals_) var_exp = [(i / tot)*100 for i in sorted(pca.e_vals_, reverse=True)] cum_var_exp = np.cumsum(pca.e_vals_normalized_*100) with plt.style.context('seaborn-whitegrid'): fig, ax = plt.subplots(figsize=(6, 4)) plt.bar(range(4), var_exp, alpha=0.5, align='center', label='individual explained variance') plt.step(range(4), cum_var_exp, where='mid', label='cumulative explained variance') plt.ylabel('Explained variance ratio') plt.xlabel('Principal components') plt.xticks(range(4)) ax.set_xticklabels(np.arange(1, X.shape[1] + 1)) plt.legend(loc='best') plt.tight_layout()","title":"Example 2 - Plotting the Variance Explained Ratio"},{"location":"user_guide/feature_extraction/PrincipalComponentAnalysis/#example-3-pca-via-svd","text":"While the eigendecomposition of the covariance or correlation matrix may be more intuitiuve, most PCA implementations perform a Singular Vector Decomposition (SVD) to improve the computational efficiency. Another advantage of using SVD is that the results tend to be more numerically stable, since we can decompose the input matrix directly without the additional covariance-matrix step. from mlxtend.data import iris_data from mlxtend.preprocessing import standardize from mlxtend.feature_extraction import PrincipalComponentAnalysis X, y = iris_data() X = standardize(X) pca = PrincipalComponentAnalysis(n_components=2, solver='svd') pca.fit(X) X_pca = pca.transform(X) import matplotlib.pyplot as plt with plt.style.context('seaborn-whitegrid'): plt.figure(figsize=(6, 4)) for lab, col in zip((0, 1, 2), ('blue', 'red', 'green')): plt.scatter(X_pca[y==lab, 0], X_pca[y==lab, 1], label=lab, c=col) plt.xlabel('Principal Component 1') plt.ylabel('Principal Component 2') plt.legend(loc='lower center') plt.tight_layout() plt.show() If we compare this PCA projection to the previous plot in example 1, we notice that they are mirror images of each other. Note that this is not due to an error in any of those two implementations, but the reason for this difference is that, depending on the eigensolver, eigenvectors can have either negative or positive signs. For instance, if v is an eigenvector of a matrix \\Sigma , we have \\Sigma v = \\lambda v, where \\lambda is our eigenvalue then -v is also an eigenvector that has the same eigenvalue, since \\Sigma(-v) = -\\Sigma v = -\\lambda v = \\lambda(-v).","title":"Example 3 - PCA via SVD"},{"location":"user_guide/feature_extraction/PrincipalComponentAnalysis/#example-4-factor-loadings","text":"After evoking the fit method, the factor loadings are available via the loadings_ attribute. In simple terms, the loadings are the unstandardized values of the eigenvectors. Or in other words, we can interpret the loadings as the covariances (or correlation in case we standardized the input features) between the input features and the principal components (or eigenvectors), which have been scaled to unit length. By having the loadings scaled, they become comparable by magnitude and we can assess how much variance in a component is attributed to the input features (as the components are just a weighted linear combination of the input features). from mlxtend.data import iris_data from mlxtend.preprocessing import standardize from mlxtend.feature_extraction import PrincipalComponentAnalysis import matplotlib.pyplot as plt X, y = iris_data() X = standardize(X) pca = PrincipalComponentAnalysis(n_components=2, solver='eigen') pca.fit(X); xlabels = ['sepal length', 'sepal width', 'petal length', 'petal width'] fig, ax = plt.subplots(1, 2, figsize=(8, 3)) ax[0].bar(range(4), pca.loadings_[:, 0], align='center') ax[1].bar(range(4), pca.loadings_[:, 1], align='center') ax[0].set_ylabel('Factor loading onto PC1') ax[1].set_ylabel('Factor loading onto PC2') ax[0].set_xticks(range(4)) ax[1].set_xticks(range(4)) ax[0].set_xticklabels(xlabels, rotation=45) ax[1].set_xticklabels(xlabels, rotation=45) plt.ylim([-1, 1]) plt.tight_layout() For instance, we may say that most of the variance in the first component is attributed to the petal features (although the loading of sepal length on PC1 is also not much less in magnitude). In contrast, the remaining variance captured by PC2 is mostly due to the sepal width. Note that we know from Example 2 that PC1 explains most of the variance, and based on the information from the loading plots, we may say that petal features combined with sepal length may explain most of the spread in the data.","title":"Example 4 - Factor Loadings"},{"location":"user_guide/feature_extraction/PrincipalComponentAnalysis/#example-5-feature-extraction-pipeline","text":"from sklearn.pipeline import make_pipeline from sklearn.neighbors import KNeighborsClassifier from sklearn.preprocessing import StandardScaler from sklearn.model_selection import train_test_split from mlxtend.data import wine_data X, y = wine_data() X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=123, test_size=0.3, stratify=y) pipe_pca = make_pipeline(StandardScaler(), PrincipalComponentAnalysis(n_components=3), KNeighborsClassifier(n_neighbors=5)) pipe_pca.fit(X_train, y_train) print('Transf. training accyracy: %.2f%%' % (pipe_pca.score(X_train, y_train)*100)) print('Transf. test accyracy: %.2f%%' % (pipe_pca.score(X_test, y_test)*100)) Transf. training accyracy: 96.77% Transf. test accyracy: 96.30%","title":"Example 5 - Feature Extraction Pipeline"},{"location":"user_guide/feature_extraction/PrincipalComponentAnalysis/#example-6-whitening","text":"Certain algorithms require the data to be whitened. This means that the features have unit variance and the off-diagonals are all zero (i.e., the features are uncorrelated). PCA already ensures that the features are uncorrelated, hence, we only need to apply a simple scaling to whiten the transformed data. For instance, for a given transformed feature X'_i , we divide it by the square-root of the corresponding eigenvalue \\lambda_i : X'_{\\text{whitened}} = \\frac{X'_i}{\\sqrt{\\lambda_i}}. The whitening via the PrincipalComponentAnalysis can be achieved by setting whitening=True during initialization. Let's demonstrate that with an example. from sklearn.preprocessing import StandardScaler from sklearn.model_selection import train_test_split from mlxtend.data import wine_data X, y = wine_data() X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=123, test_size=0.3, stratify=y)","title":"Example 6 - Whitening"},{"location":"user_guide/feature_extraction/PrincipalComponentAnalysis/#regular-pca","text":"sc = StandardScaler() pca1 = PrincipalComponentAnalysis(n_components=2) X_train_scaled = sc.fit_transform(X_train) X_train_transf = pca1.fit(X_train_scaled).transform(X_train_scaled) with plt.style.context('seaborn-whitegrid'): plt.figure(figsize=(6, 4)) for lab, col in zip((0, 1, 2), ('blue', 'red', 'green')): plt.scatter(X_train_transf[y_train==lab, 0], X_train_transf[y_train==lab, 1], label=lab, c=col) plt.xlabel('Principal Component 1') plt.ylabel('Principal Component 2') plt.legend(loc='lower center') plt.tight_layout() plt.show() np.set_printoptions(precision=1, suppress=True) print('Covariance matrix:\\n') np.cov(X_train_transf.T) Covariance matrix: array([[4.9, 0. ], [0. , 2.5]]) As we can see, the features are uncorrelated after transformation but don't have unit variance.","title":"Regular PCA"},{"location":"user_guide/feature_extraction/PrincipalComponentAnalysis/#pca-with-whitening","text":"sc = StandardScaler() pca1 = PrincipalComponentAnalysis(n_components=2, whitening=True) X_train_scaled = sc.fit_transform(X_train) X_train_transf = pca1.fit(X_train_scaled).transform(X_train_scaled) with plt.style.context('seaborn-whitegrid'): plt.figure(figsize=(6, 4)) for lab, col in zip((0, 1, 2), ('blue', 'red', 'green')): plt.scatter(X_train_transf[y_train==lab, 0], X_train_transf[y_train==lab, 1], label=lab, c=col) plt.xlabel('Principal Component 1') plt.ylabel('Principal Component 2') plt.legend(loc='lower center') plt.tight_layout() plt.show() np.set_printoptions(precision=1, suppress=True) print('Covariance matrix:\\n') np.cov(X_train_transf.T) Covariance matrix: array([[1., 0.], [0., 1.]]) As we can see above, the whitening achieves that all features now have unit variance. I.e., the covariance matrix of the transformed features becomes the identity matrix.","title":"PCA with Whitening"},{"location":"user_guide/feature_extraction/PrincipalComponentAnalysis/#api","text":"PrincipalComponentAnalysis(n_components=None, solver='svd', whitening=False) Principal Component Analysis Class Parameters n_components : int (default: None) The number of principal components for transformation. Keeps the original dimensions of the dataset if None . solver : str (default: 'svd') Method for performing the matrix decomposition. {'eigen', 'svd'} whitening : bool (default: False) Performs whitening such that the covariance matrix of the transformed data will be the identity matrix. Attributes w_ : array-like, shape=[n_features, n_components] Projection matrix e_vals_ : array-like, shape=[n_features] Eigenvalues in sorted order. e_vecs_ : array-like, shape=[n_features] Eigenvectors in sorted order. e_vals_normalized_ : array-like, shape=[n_features] Normalized eigen values such that they sum up to 1. This is equal to what's often referred to as \"explained variance ratios.\" loadings_ : array_like, shape=[n_features, n_features] The factor loadings of the original variables onto the principal components. The columns are the principal components, and the rows are the features loadings. For instance, the first column contains the loadings onto the first principal component. Note that the signs may be flipped depending on whether you use the 'eigen' or 'svd' solver; this does not affect the interpretation of the loadings though. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/feature_extraction/PrincipalComponentAnalysis/","title":"API"},{"location":"user_guide/feature_extraction/PrincipalComponentAnalysis/#methods","text":"fit(X, y=None) Learn model from training data. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns self : object get_params(deep=True) Get parameters for this estimator. Parameters deep : boolean, optional If True, will return the parameters for this estimator and contained subobjects that are estimators. Returns params : mapping of string to any Parameter names mapped to their values.' adapted from https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/base.py Author: Gael Varoquaux gael.varoquaux@normalesup.org License: BSD 3 clause set_params( params) Set the parameters of this estimator. The method works on simple estimators as well as on nested objects (such as pipelines). The latter have parameters of the form <component>__<parameter> so that it's possible to update each component of a nested object. Returns self adapted from https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/base.py Author: Gael Varoquaux gael.varoquaux@normalesup.org License: BSD 3 clause transform(X) Apply the linear transformation on X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns X_projected : np.ndarray, shape = [n_samples, n_components] Projected training vectors.","title":"Methods"},{"location":"user_guide/feature_extraction/RBFKernelPCA/","text":"RBFKernelPCA: RBF kernel principal component analysis Implementation of RBF Kernel Principal Component Analysis for non-linear dimensionality reduction from mlxtend.feature_extraction import RBFKernelPCA Overview Most machine learning algorithms have been developed and statistically validated for linearly separable data. Popular examples are linear classifiers like Support Vector Machines (SVMs) or the (standard) Principal Component Analysis (PCA) for dimensionality reduction. However, most real world data requires nonlinear methods in order to perform tasks that involve the analysis and discovery of patterns successfully. The focus of this overview is to briefly introduce the idea of kernel methods and to implement a Gaussian radius basis function (RBF) kernel that is used to perform nonlinear dimensionality reduction via BF kernel principal component analysis (kPCA). Principal Component Analysis The main purpose of principal component analysis (PCA) is the analysis of data to identify patterns that represent the data \u201cwell.\u201d The principal components can be understood as new axes of the dataset that maximize the variance along those axes (the eigenvectors of the covariance matrix). In other words, PCA aims to find the axes with maximum variances along which the data is most spread. For more details, please see the related article on mlxtend.feature_extraction.PrincipalComponentAnalysis . Nonlinear dimensionality reduction The \u201cclassic\u201d PCA approach described above is a linear projection technique that works well if the data is linearly separable. However, in the case of linearly inseparable data, a nonlinear technique is required if the task is to reduce the dimensionality of a dataset. Kernel functions and the kernel trick The basic idea to deal with linearly inseparable data is to project it onto a higher dimensional space where it becomes linearly separable. Let us call this nonlinear mapping function \\phi so that the mapping of a sample \\mathbf{x} can be written as \\mathbf{x} \\rightarrow \\phi (\\mathbf{x}) , which is called \"kernel function.\" Now, the term \"kernel\" describes a function that calculates the dot product of the images of the samples \\mathbf{x} under \\phi . \\kappa(\\mathbf{x_i, x_j}) = \\phi (\\mathbf{x_i}) \\phi (\\mathbf{x_j})^T More details about the derivation of this equation are provided in this excellent review article by Quan Wang: Kernel Principal Component Analysis and its Applications in Face Recognition and Active Shape Models .[ 1 ] In other words, the function \\phi maps the original d-dimensional features into a larger, k-dimensional feature space by creating nononlinear combinations of the original features. For example, if \\mathbf{x} consists of 2 features: \\mathbf{x} = \\big[x_1 \\quad x_2\\big]^T \\quad \\quad \\mathbf{x} \\in I\\!R^d \\Downarrow \\phi \\mathbf{x}' = \\big[x_1 \\quad x_2 \\quad x_1 x_2 \\quad x_{1}^2 \\quad x_1 x_{2}^3 \\quad \\dots \\big]^T \\quad \\quad \\mathbf{x} \\in I\\!R^k (k >> d) Often, the mathematical definition of the RBF kernel is written and implemented as \\kappa(\\mathbf{x_i, x_j}) = exp\\bigg(- \\gamma \\; \\lVert\\mathbf{x_i - x_j }\\rVert^{2}_{2} \\bigg) where \\textstyle\\gamma = \\tfrac{1}{2\\sigma^2} is a free parameter that is to be optimized. Gaussian radial basis function (RBF) Kernel PCA In the linear PCA approach, we are interested in the principal components that maximize the variance in the dataset. This is done by extracting the eigenvectors (principle components) that correspond to the largest eigenvalues based on the covariance matrix: \\text{Cov} = \\frac{1}{N} \\sum_{i=1}^{N} \\mathbf{x_i} \\mathbf{x_i}^T Bernhard Scholkopf ( Kernel Principal Component Analysis [ 2 ]) generalized this approach for data that was mapped onto the higher dimensional space via a kernel function: \\text{Cov} = \\frac{1}{N} \\sum_{i=1}^{N} \\phi(\\mathbf{x_i}) \\phi(\\mathbf{x_i})^T However, in practice the the covariance matrix in the higher dimensional space is not calculated explicitly (kernel trick). Therefore, the implementation of RBF kernel PCA does not yield the principal component axes (in contrast to the standard PCA), but the obtained eigenvectors can be understood as projections of the data onto the principal components. RBF kernel PCA step-by-step 1. Computation of the kernel (similarity) matrix. In this first step, we need to calculate \\kappa(\\mathbf{x_i, x_j}) = exp\\bigg(- \\gamma \\; \\lVert\\mathbf{x_i - x_j }\\rVert^{2}_{2} \\bigg) for every pair of points. E.g., if we have a dataset of 100 samples, this step would result in a symmetric 100x100 kernel matrix. 2. Eigendecomposition of the kernel matrix. Since it is not guaranteed that the kernel matrix is centered, we can apply the following equation to do so: K' = K - \\mathbf{1_N} K - K \\mathbf{1_N} + \\mathbf{1_N} K \\mathbf{1_N} where \\mathbf{1_N} is (like the kernel matrix) a N\\times N matrix with all values equal to \\frac{1}{N} . [ 3 ] Now, we have to obtain the eigenvectors of the centered kernel matrix that correspond to the largest eigenvalues. Those eigenvectors are the data points already projected onto the respective principal components. Projecting new data So far, so good, in the sections above, we have been projecting an dataset onto a new feature subspace. However, in a real application, we are usually interested in mapping new data points onto the same new feature subspace (e.g., if are working with a training and a test dataset in pattern classification tasks). Remember, when we computed the eigenvectors \\mathbf{\\alpha} of the centered kernel matrix, those values were actually already the projected datapoints onto the principal component axis \\mathbf{g} . If we want to project a new data point \\mathbf{x} onto this principal component axis, we'd need to compute \\phi(\\mathbf{x})^T \\mathbf{g} . Fortunately, also here, we don't have to compute \\phi(\\mathbf{x})^T \\mathbf{g} explicitely but use the kernel trick to calculate the RBF kernel between the new data point and every data point j in the training dataset: \\phi(\\mathbf{x})^T \\mathbf{g} = \\sum_j \\alpha_{i} \\; \\phi(\\mathbf{x}) \\; \\phi(\\mathbf{x_j})^T = \\sum_j \\alpha_{i} \\; \\kappa(\\mathbf{x}, \\mathbf{x_j}) and the eigenvectors \\alpha and eigenvalues \\lambda of the Kernel matrix \\mathbf{K} satisfy the equation \\mathbf{K} \\alpha = \\lambda \\alpha , we just need to normalize the eigenvector by the corresponding eigenvalue. References [1] Q. Wang. Kernel principal component analysis and its applications in face recognition and active shape models . CoRR, abs/1207.3538, 2012. [2] B. Scholkopf, A. Smola, and K.-R. Muller. Kernel principal component analysis . pages 583\u2013588, 1997. [3] B. Scholkopf, A. Smola, and K.-R. Muller. Nonlinear component analysis as a kernel eigenvalue problem . Neural computation, 10(5):1299\u20131319, 1998. Example 1 - Half-moon shapes We will start with a simple example of 2 half-moon shapes generated by the make_moons function from scikit-learn. import matplotlib.pyplot as plt from sklearn.datasets import make_moons X, y = make_moons(n_samples=50, random_state=1) plt.scatter(X[y==0, 0], X[y==0, 1], color='red', marker='o', alpha=0.5) plt.scatter(X[y==1, 0], X[y==1, 1], color='blue', marker='^', alpha=0.5) plt.ylabel('y coordinate') plt.xlabel('x coordinate') plt.show() Since the two half-moon shapes are linearly inseparable, we expect that the \u201cclassic\u201d PCA will fail to give us a \u201cgood\u201d representation of the data in 1D space. Let us use PCA class to perform the dimensionality reduction. from mlxtend.feature_extraction import PrincipalComponentAnalysis as PCA pca = PCA(n_components=2) X_pca = pca.fit(X).transform(X) plt.scatter(X_pca[y==0, 0], X_pca[y==0, 1], color='red', marker='o', alpha=0.5) plt.scatter(X_pca[y==1, 0], X_pca[y==1, 1], color='blue', marker='^', alpha=0.5) plt.xlabel('PC1') plt.ylabel('PC2') plt.show() As we can see, the resulting principal components do not yield a subspace where the data is linearly separated well. Note that PCA is a unsupervised method and does not \u201cconsider\u201d class labels in order to maximize the variance in contrast to Linear Discriminant Analysis. Here, the colors blue and red are just added for visualization purposes to indicate the degree of separation. Next, we will perform dimensionality reduction via RBF kernel PCA on our half-moon data. The choice of \\gamma depends on the dataset and can be obtained via hyperparameter tuning techniques like Grid Search. Hyperparameter tuning is a broad topic itself, and here I will just use a \\gamma -value that I found to produce \u201cgood\u201d results. from mlxtend.data import iris_data from mlxtend.preprocessing import standardize from mlxtend.feature_extraction import RBFKernelPCA as KPCA kpca = KPCA(gamma=15.0, n_components=2) kpca.fit(X) X_kpca = kpca.X_projected_ Please note that the components of kernel methods such as RBF kernel PCA already represent the projected data points (in contrast to PCA, where the component axis are the \"top k\" eigenvectors thar are used to contruct a projection matrix, which is then used to transform the training samples). Thus, the projected training set is available after fitting via the .X_projected_ attribute. plt.scatter(X_kpca[y==0, 0], X_kpca[y==0, 1], color='red', marker='o', alpha=0.5) plt.scatter(X_kpca[y==1, 0], X_kpca[y==1, 1], color='blue', marker='^', alpha=0.5) plt.title('First 2 principal components after RBF Kernel PCA') plt.xlabel('PC1') plt.ylabel('PC2') plt.show() The new feature space is linearly separable now. Since we are often interested in dimensionality reduction, let's have a look at the first component only. import numpy as np plt.scatter(X_kpca[y==0, 0], np.zeros((25, 1)), color='red', marker='o', alpha=0.5) plt.scatter(X_kpca[y==1, 0], np.zeros((25, 1)), color='blue', marker='^', alpha=0.5) plt.title('First principal component after RBF Kernel PCA') plt.xlabel('PC1') plt.yticks([]) plt.show() We can clearly see that the projection via RBF kernel PCA yielded a subspace where the classes are separated well. Such a subspace can then be used as input for generalized linear classification models, e.g., logistic regression. Projecting new data Finally, via the transform method, we can project new data onto the new component axes. import matplotlib.pyplot as plt from sklearn.datasets import make_moons X2, y2 = make_moons(n_samples=200, random_state=5) X2_kpca = kpca.transform(X2) plt.scatter(X_kpca[y==0, 0], X_kpca[y==0, 1], color='red', marker='o', alpha=0.5, label='fit data') plt.scatter(X_kpca[y==1, 0], X_kpca[y==1, 1], color='blue', marker='^', alpha=0.5, label='fit data') plt.scatter(X2_kpca[y2==0, 0], X2_kpca[y2==0, 1], color='orange', marker='v', alpha=0.2, label='new data') plt.scatter(X2_kpca[y2==1, 0], X2_kpca[y2==1, 1], color='cyan', marker='s', alpha=0.2, label='new data') plt.legend() plt.show() Example 2 - Concentric circles Following the concepts explained in example 1, let's have a look at another classic case: 2 concentric circles with random noise produced by scikit-learn\u2019s make_circles . from sklearn.datasets import make_circles X, y = make_circles(n_samples=1000, random_state=123, noise=0.1, factor=0.2) plt.figure(figsize=(8,6)) plt.scatter(X[y==0, 0], X[y==0, 1], color='red', alpha=0.5) plt.scatter(X[y==1, 0], X[y==1, 1], color='blue', alpha=0.5) plt.title('Concentric circles') plt.ylabel('y coordinate') plt.xlabel('x coordinate') plt.show() from mlxtend.data import iris_data from mlxtend.preprocessing import standardize from mlxtend.feature_extraction import RBFKernelPCA as KPCA kpca = KPCA(gamma=15.0, n_components=2) kpca.fit(X) X_kpca = kpca.X_projected_ plt.scatter(X_kpca[y==0, 0], X_kpca[y==0, 1], color='red', marker='o', alpha=0.5) plt.scatter(X_kpca[y==1, 0], X_kpca[y==1, 1], color='blue', marker='^', alpha=0.5) plt.title('First 2 principal components after RBF Kernel PCA') plt.xlabel('PC1') plt.ylabel('PC2') plt.show() plt.scatter(X_kpca[y==0, 0], np.zeros((500, 1)), color='red', marker='o', alpha=0.5) plt.scatter(X_kpca[y==1, 0], np.zeros((500, 1)), color='blue', marker='^', alpha=0.5) plt.title('First principal component after RBF Kernel PCA') plt.xlabel('PC1') plt.yticks([]) plt.show() API RBFKernelPCA(gamma=15.0, n_components=None, copy_X=True) RBF Kernel Principal Component Analysis for dimensionality reduction. Parameters gamma : float (default: 15.0) Free parameter (coefficient) of the RBF kernel. n_components : int (default: None) The number of principal components for transformation. Keeps the original dimensions of the dataset if None . copy_X : bool (default: True) Copies training data, which is required to compute the projection of new data via the transform method. Uses a reference to X if False. Attributes e_vals_ : array-like, shape=[n_features] Eigenvalues in sorted order. e_vecs_ : array-like, shape=[n_features] Eigenvectors in sorted order. X_projected_ : array-like, shape=[n_samples, n_components] Training samples projected along the component axes. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/feature_extraction/RBFKernelPCA/ Methods fit(X) Learn model from training data. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns self : object transform(X) Apply the non-linear transformation on X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns X_projected : np.ndarray, shape = [n_samples, n_components] Projected training vectors.","title":"RBFKernelPCA"},{"location":"user_guide/feature_extraction/RBFKernelPCA/#rbfkernelpca-rbf-kernel-principal-component-analysis","text":"Implementation of RBF Kernel Principal Component Analysis for non-linear dimensionality reduction from mlxtend.feature_extraction import RBFKernelPCA","title":"RBFKernelPCA: RBF kernel principal component analysis"},{"location":"user_guide/feature_extraction/RBFKernelPCA/#overview","text":"Most machine learning algorithms have been developed and statistically validated for linearly separable data. Popular examples are linear classifiers like Support Vector Machines (SVMs) or the (standard) Principal Component Analysis (PCA) for dimensionality reduction. However, most real world data requires nonlinear methods in order to perform tasks that involve the analysis and discovery of patterns successfully. The focus of this overview is to briefly introduce the idea of kernel methods and to implement a Gaussian radius basis function (RBF) kernel that is used to perform nonlinear dimensionality reduction via BF kernel principal component analysis (kPCA).","title":"Overview"},{"location":"user_guide/feature_extraction/RBFKernelPCA/#principal-component-analysis","text":"The main purpose of principal component analysis (PCA) is the analysis of data to identify patterns that represent the data \u201cwell.\u201d The principal components can be understood as new axes of the dataset that maximize the variance along those axes (the eigenvectors of the covariance matrix). In other words, PCA aims to find the axes with maximum variances along which the data is most spread. For more details, please see the related article on mlxtend.feature_extraction.PrincipalComponentAnalysis .","title":"Principal Component Analysis"},{"location":"user_guide/feature_extraction/RBFKernelPCA/#nonlinear-dimensionality-reduction","text":"The \u201cclassic\u201d PCA approach described above is a linear projection technique that works well if the data is linearly separable. However, in the case of linearly inseparable data, a nonlinear technique is required if the task is to reduce the dimensionality of a dataset.","title":"Nonlinear dimensionality reduction"},{"location":"user_guide/feature_extraction/RBFKernelPCA/#kernel-functions-and-the-kernel-trick","text":"The basic idea to deal with linearly inseparable data is to project it onto a higher dimensional space where it becomes linearly separable. Let us call this nonlinear mapping function \\phi so that the mapping of a sample \\mathbf{x} can be written as \\mathbf{x} \\rightarrow \\phi (\\mathbf{x}) , which is called \"kernel function.\" Now, the term \"kernel\" describes a function that calculates the dot product of the images of the samples \\mathbf{x} under \\phi . \\kappa(\\mathbf{x_i, x_j}) = \\phi (\\mathbf{x_i}) \\phi (\\mathbf{x_j})^T More details about the derivation of this equation are provided in this excellent review article by Quan Wang: Kernel Principal Component Analysis and its Applications in Face Recognition and Active Shape Models .[ 1 ] In other words, the function \\phi maps the original d-dimensional features into a larger, k-dimensional feature space by creating nononlinear combinations of the original features. For example, if \\mathbf{x} consists of 2 features: \\mathbf{x} = \\big[x_1 \\quad x_2\\big]^T \\quad \\quad \\mathbf{x} \\in I\\!R^d \\Downarrow \\phi \\mathbf{x}' = \\big[x_1 \\quad x_2 \\quad x_1 x_2 \\quad x_{1}^2 \\quad x_1 x_{2}^3 \\quad \\dots \\big]^T \\quad \\quad \\mathbf{x} \\in I\\!R^k (k >> d) Often, the mathematical definition of the RBF kernel is written and implemented as \\kappa(\\mathbf{x_i, x_j}) = exp\\bigg(- \\gamma \\; \\lVert\\mathbf{x_i - x_j }\\rVert^{2}_{2} \\bigg) where \\textstyle\\gamma = \\tfrac{1}{2\\sigma^2} is a free parameter that is to be optimized.","title":"Kernel functions and the kernel trick"},{"location":"user_guide/feature_extraction/RBFKernelPCA/#gaussian-radial-basis-function-rbf-kernel-pca","text":"In the linear PCA approach, we are interested in the principal components that maximize the variance in the dataset. This is done by extracting the eigenvectors (principle components) that correspond to the largest eigenvalues based on the covariance matrix: \\text{Cov} = \\frac{1}{N} \\sum_{i=1}^{N} \\mathbf{x_i} \\mathbf{x_i}^T Bernhard Scholkopf ( Kernel Principal Component Analysis [ 2 ]) generalized this approach for data that was mapped onto the higher dimensional space via a kernel function: \\text{Cov} = \\frac{1}{N} \\sum_{i=1}^{N} \\phi(\\mathbf{x_i}) \\phi(\\mathbf{x_i})^T However, in practice the the covariance matrix in the higher dimensional space is not calculated explicitly (kernel trick). Therefore, the implementation of RBF kernel PCA does not yield the principal component axes (in contrast to the standard PCA), but the obtained eigenvectors can be understood as projections of the data onto the principal components.","title":"Gaussian radial basis function (RBF) Kernel PCA"},{"location":"user_guide/feature_extraction/RBFKernelPCA/#rbf-kernel-pca-step-by-step","text":"","title":"RBF kernel PCA step-by-step"},{"location":"user_guide/feature_extraction/RBFKernelPCA/#1-computation-of-the-kernel-similarity-matrix","text":"In this first step, we need to calculate \\kappa(\\mathbf{x_i, x_j}) = exp\\bigg(- \\gamma \\; \\lVert\\mathbf{x_i - x_j }\\rVert^{2}_{2} \\bigg) for every pair of points. E.g., if we have a dataset of 100 samples, this step would result in a symmetric 100x100 kernel matrix.","title":"1. Computation of the kernel (similarity) matrix."},{"location":"user_guide/feature_extraction/RBFKernelPCA/#2-eigendecomposition-of-the-kernel-matrix","text":"Since it is not guaranteed that the kernel matrix is centered, we can apply the following equation to do so: K' = K - \\mathbf{1_N} K - K \\mathbf{1_N} + \\mathbf{1_N} K \\mathbf{1_N} where \\mathbf{1_N} is (like the kernel matrix) a N\\times N matrix with all values equal to \\frac{1}{N} . [ 3 ] Now, we have to obtain the eigenvectors of the centered kernel matrix that correspond to the largest eigenvalues. Those eigenvectors are the data points already projected onto the respective principal components.","title":"2. Eigendecomposition of the kernel matrix."},{"location":"user_guide/feature_extraction/RBFKernelPCA/#projecting-new-data","text":"So far, so good, in the sections above, we have been projecting an dataset onto a new feature subspace. However, in a real application, we are usually interested in mapping new data points onto the same new feature subspace (e.g., if are working with a training and a test dataset in pattern classification tasks). Remember, when we computed the eigenvectors \\mathbf{\\alpha} of the centered kernel matrix, those values were actually already the projected datapoints onto the principal component axis \\mathbf{g} . If we want to project a new data point \\mathbf{x} onto this principal component axis, we'd need to compute \\phi(\\mathbf{x})^T \\mathbf{g} . Fortunately, also here, we don't have to compute \\phi(\\mathbf{x})^T \\mathbf{g} explicitely but use the kernel trick to calculate the RBF kernel between the new data point and every data point j in the training dataset: \\phi(\\mathbf{x})^T \\mathbf{g} = \\sum_j \\alpha_{i} \\; \\phi(\\mathbf{x}) \\; \\phi(\\mathbf{x_j})^T = \\sum_j \\alpha_{i} \\; \\kappa(\\mathbf{x}, \\mathbf{x_j}) and the eigenvectors \\alpha and eigenvalues \\lambda of the Kernel matrix \\mathbf{K} satisfy the equation \\mathbf{K} \\alpha = \\lambda \\alpha , we just need to normalize the eigenvector by the corresponding eigenvalue.","title":"Projecting new data"},{"location":"user_guide/feature_extraction/RBFKernelPCA/#references","text":"[1] Q. Wang. Kernel principal component analysis and its applications in face recognition and active shape models . CoRR, abs/1207.3538, 2012. [2] B. Scholkopf, A. Smola, and K.-R. Muller. Kernel principal component analysis . pages 583\u2013588, 1997. [3] B. Scholkopf, A. Smola, and K.-R. Muller. Nonlinear component analysis as a kernel eigenvalue problem . Neural computation, 10(5):1299\u20131319, 1998.","title":"References"},{"location":"user_guide/feature_extraction/RBFKernelPCA/#example-1-half-moon-shapes","text":"We will start with a simple example of 2 half-moon shapes generated by the make_moons function from scikit-learn. import matplotlib.pyplot as plt from sklearn.datasets import make_moons X, y = make_moons(n_samples=50, random_state=1) plt.scatter(X[y==0, 0], X[y==0, 1], color='red', marker='o', alpha=0.5) plt.scatter(X[y==1, 0], X[y==1, 1], color='blue', marker='^', alpha=0.5) plt.ylabel('y coordinate') plt.xlabel('x coordinate') plt.show() Since the two half-moon shapes are linearly inseparable, we expect that the \u201cclassic\u201d PCA will fail to give us a \u201cgood\u201d representation of the data in 1D space. Let us use PCA class to perform the dimensionality reduction. from mlxtend.feature_extraction import PrincipalComponentAnalysis as PCA pca = PCA(n_components=2) X_pca = pca.fit(X).transform(X) plt.scatter(X_pca[y==0, 0], X_pca[y==0, 1], color='red', marker='o', alpha=0.5) plt.scatter(X_pca[y==1, 0], X_pca[y==1, 1], color='blue', marker='^', alpha=0.5) plt.xlabel('PC1') plt.ylabel('PC2') plt.show() As we can see, the resulting principal components do not yield a subspace where the data is linearly separated well. Note that PCA is a unsupervised method and does not \u201cconsider\u201d class labels in order to maximize the variance in contrast to Linear Discriminant Analysis. Here, the colors blue and red are just added for visualization purposes to indicate the degree of separation. Next, we will perform dimensionality reduction via RBF kernel PCA on our half-moon data. The choice of \\gamma depends on the dataset and can be obtained via hyperparameter tuning techniques like Grid Search. Hyperparameter tuning is a broad topic itself, and here I will just use a \\gamma -value that I found to produce \u201cgood\u201d results. from mlxtend.data import iris_data from mlxtend.preprocessing import standardize from mlxtend.feature_extraction import RBFKernelPCA as KPCA kpca = KPCA(gamma=15.0, n_components=2) kpca.fit(X) X_kpca = kpca.X_projected_ Please note that the components of kernel methods such as RBF kernel PCA already represent the projected data points (in contrast to PCA, where the component axis are the \"top k\" eigenvectors thar are used to contruct a projection matrix, which is then used to transform the training samples). Thus, the projected training set is available after fitting via the .X_projected_ attribute. plt.scatter(X_kpca[y==0, 0], X_kpca[y==0, 1], color='red', marker='o', alpha=0.5) plt.scatter(X_kpca[y==1, 0], X_kpca[y==1, 1], color='blue', marker='^', alpha=0.5) plt.title('First 2 principal components after RBF Kernel PCA') plt.xlabel('PC1') plt.ylabel('PC2') plt.show() The new feature space is linearly separable now. Since we are often interested in dimensionality reduction, let's have a look at the first component only. import numpy as np plt.scatter(X_kpca[y==0, 0], np.zeros((25, 1)), color='red', marker='o', alpha=0.5) plt.scatter(X_kpca[y==1, 0], np.zeros((25, 1)), color='blue', marker='^', alpha=0.5) plt.title('First principal component after RBF Kernel PCA') plt.xlabel('PC1') plt.yticks([]) plt.show() We can clearly see that the projection via RBF kernel PCA yielded a subspace where the classes are separated well. Such a subspace can then be used as input for generalized linear classification models, e.g., logistic regression.","title":"Example 1 - Half-moon shapes"},{"location":"user_guide/feature_extraction/RBFKernelPCA/#projecting-new-data_1","text":"Finally, via the transform method, we can project new data onto the new component axes. import matplotlib.pyplot as plt from sklearn.datasets import make_moons X2, y2 = make_moons(n_samples=200, random_state=5) X2_kpca = kpca.transform(X2) plt.scatter(X_kpca[y==0, 0], X_kpca[y==0, 1], color='red', marker='o', alpha=0.5, label='fit data') plt.scatter(X_kpca[y==1, 0], X_kpca[y==1, 1], color='blue', marker='^', alpha=0.5, label='fit data') plt.scatter(X2_kpca[y2==0, 0], X2_kpca[y2==0, 1], color='orange', marker='v', alpha=0.2, label='new data') plt.scatter(X2_kpca[y2==1, 0], X2_kpca[y2==1, 1], color='cyan', marker='s', alpha=0.2, label='new data') plt.legend() plt.show()","title":"Projecting new data"},{"location":"user_guide/feature_extraction/RBFKernelPCA/#example-2-concentric-circles","text":"Following the concepts explained in example 1, let's have a look at another classic case: 2 concentric circles with random noise produced by scikit-learn\u2019s make_circles . from sklearn.datasets import make_circles X, y = make_circles(n_samples=1000, random_state=123, noise=0.1, factor=0.2) plt.figure(figsize=(8,6)) plt.scatter(X[y==0, 0], X[y==0, 1], color='red', alpha=0.5) plt.scatter(X[y==1, 0], X[y==1, 1], color='blue', alpha=0.5) plt.title('Concentric circles') plt.ylabel('y coordinate') plt.xlabel('x coordinate') plt.show() from mlxtend.data import iris_data from mlxtend.preprocessing import standardize from mlxtend.feature_extraction import RBFKernelPCA as KPCA kpca = KPCA(gamma=15.0, n_components=2) kpca.fit(X) X_kpca = kpca.X_projected_ plt.scatter(X_kpca[y==0, 0], X_kpca[y==0, 1], color='red', marker='o', alpha=0.5) plt.scatter(X_kpca[y==1, 0], X_kpca[y==1, 1], color='blue', marker='^', alpha=0.5) plt.title('First 2 principal components after RBF Kernel PCA') plt.xlabel('PC1') plt.ylabel('PC2') plt.show() plt.scatter(X_kpca[y==0, 0], np.zeros((500, 1)), color='red', marker='o', alpha=0.5) plt.scatter(X_kpca[y==1, 0], np.zeros((500, 1)), color='blue', marker='^', alpha=0.5) plt.title('First principal component after RBF Kernel PCA') plt.xlabel('PC1') plt.yticks([]) plt.show()","title":"Example 2 - Concentric circles"},{"location":"user_guide/feature_extraction/RBFKernelPCA/#api","text":"RBFKernelPCA(gamma=15.0, n_components=None, copy_X=True) RBF Kernel Principal Component Analysis for dimensionality reduction. Parameters gamma : float (default: 15.0) Free parameter (coefficient) of the RBF kernel. n_components : int (default: None) The number of principal components for transformation. Keeps the original dimensions of the dataset if None . copy_X : bool (default: True) Copies training data, which is required to compute the projection of new data via the transform method. Uses a reference to X if False. Attributes e_vals_ : array-like, shape=[n_features] Eigenvalues in sorted order. e_vecs_ : array-like, shape=[n_features] Eigenvectors in sorted order. X_projected_ : array-like, shape=[n_samples, n_components] Training samples projected along the component axes. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/feature_extraction/RBFKernelPCA/","title":"API"},{"location":"user_guide/feature_extraction/RBFKernelPCA/#methods","text":"fit(X) Learn model from training data. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns self : object transform(X) Apply the non-linear transformation on X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns X_projected : np.ndarray, shape = [n_samples, n_components] Projected training vectors.","title":"Methods"},{"location":"user_guide/feature_selection/ColumnSelector/","text":"ColumnSelector: Scikit-learn utility function to select specific columns in a pipeline Implementation of a column selector class for scikit-learn pipelines. from mlxtend.feature_selection import ColumnSelector Overview The ColumnSelector can be used for \"manual\" feature selection, e.g., as part of a grid search via a scikit-learn pipeline. References - Example 1 - Fitting an Estimator on a Feature Subset Load a simple benchmark dataset: from sklearn.datasets import load_iris iris = load_iris() X = iris.data y = iris.target The ColumnSelector is a simple transformer class that selects specific columns (features) from a datast. For instance, using the transform method returns a reduced dataset that only contains two features (here: the first two features via the indices 0 and 1, respectively): from mlxtend.feature_selection import ColumnSelector col_selector = ColumnSelector(cols=(0, 1)) # col_selector.fit(X) # optional, does not do anything col_selector.transform(X).shape (150, 2) ColumnSelector works both with numpy arrays and pandas dataframes: import pandas as pd iris_df = pd.DataFrame(iris.data, columns=iris.feature_names) iris_df.head() .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } sepal length (cm) sepal width (cm) petal length (cm) petal width (cm) 0 5.1 3.5 1.4 0.2 1 4.9 3.0 1.4 0.2 2 4.7 3.2 1.3 0.2 3 4.6 3.1 1.5 0.2 4 5.0 3.6 1.4 0.2 col_selector = ColumnSelector(cols=(\"sepal length (cm)\", \"sepal width (cm)\")) col_selector.transform(iris_df).shape (150, 2) Similarly, we can use the ColumnSelector as part of a scikit-learn Pipeline : from sklearn.neighbors import KNeighborsClassifier from sklearn.preprocessing import StandardScaler from sklearn.pipeline import make_pipeline pipe = make_pipeline(StandardScaler(), ColumnSelector(cols=(0, 1)), KNeighborsClassifier()) pipe.fit(X, y) pipe.score(X, y) 0.84 Example 2 - Feature Selection via GridSearch Example 1 showed a simple useage example of the ColumnSelector ; however, selecting columns from a dataset is trivial and does not require a specific transformer class since we could have achieved the same results via classifier.fit(X[:, :2], y) classifier.score(X[:, :2], y) However, the ColumnSelector becomes really useful for feature selection as part of a grid search as shown in this example. Load a simple benchmark dataset: from sklearn.datasets import load_iris iris = load_iris() X = iris.data y = iris.target Create all possible combinations: from itertools import combinations all_comb = [] for size in range(1, 5): all_comb += list(combinations(range(X.shape[1]), r=size)) print(all_comb) [(0,), (1,), (2,), (3,), (0, 1), (0, 2), (0, 3), (1, 2), (1, 3), (2, 3), (0, 1, 2), (0, 1, 3), (0, 2, 3), (1, 2, 3), (0, 1, 2, 3)] Feature and model selection via grid search: from mlxtend.feature_selection import ColumnSelector from sklearn.neighbors import KNeighborsClassifier from sklearn.preprocessing import StandardScaler from sklearn.model_selection import GridSearchCV from sklearn.pipeline import make_pipeline pipe = make_pipeline(StandardScaler(), ColumnSelector(), KNeighborsClassifier()) param_grid = {'columnselector__cols': all_comb, 'kneighborsclassifier__n_neighbors': list(range(1, 11))} grid = GridSearchCV(pipe, param_grid, cv=5, n_jobs=-1) grid.fit(X, y) print('Best parameters:', grid.best_params_) print('Best performance:', grid.best_score_) Best parameters: {'columnselector__cols': (2, 3), 'kneighborsclassifier__n_neighbors': 1} Best performance: 0.98 Example 3 -- Scaling of a Subset of Features in a scikit-learn Pipeline The following example illustrates how we could use the ColumnSelector in tandem with scikit-learn's FeatureUnion to only scale certain features (in this toy example: the first and second feature only) in a datasets in a Pipeline . from mlxtend.feature_selection import ColumnSelector from sklearn.pipeline import make_pipeline from sklearn.pipeline import Pipeline from sklearn.pipeline import FeatureUnion from sklearn.preprocessing import MinMaxScaler from sklearn.neighbors import KNeighborsClassifier from mlxtend.data import iris_data X, y = iris_data() scale_pipe = make_pipeline(ColumnSelector(cols=(0, 1)), MinMaxScaler()) pipeline = Pipeline([ ('feats', FeatureUnion([ ('col_1-2', scale_pipe), ('col_3-4', ColumnSelector(cols=(2, 3))) ])), ('clf', KNeighborsClassifier()) ]) pipeline.fit(X, y) Pipeline(memory=None, steps=[('feats', FeatureUnion(n_jobs=None, transformer_list=[('col_1-2', Pipeline(memory=None, steps=[('columnselector', ColumnSelector(cols=(0, 1), drop_axis=False)), ('minmaxscaler', MinMaxScaler(copy=True, feature_range=(0, 1)))])), ('col_3-4', ColumnSelector(cols=(2, 3), drop_axis=Fa...ki', metric_params=None, n_jobs=None, n_neighbors=5, p=2, weights='uniform'))]) API ColumnSelector(cols=None, drop_axis=False) Object for selecting specific columns from a data set. Parameters cols : array-like (default: None) A list specifying the feature indices to be selected. For example, [1, 4, 5] to select the 2nd, 5th, and 6th feature columns. If None, returns all columns in the array. drop_axis : bool (default=False) Drops last axis if True and the only one column is selected. This is useful, e.g., when the ColumnSelector is used for selecting only one column and the resulting array should be fed to e.g., a scikit-learn column selector. E.g., instead of returning an array with shape (n_samples, 1), drop_axis=True will return an aray with shape (n_samples,). Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/feature_selection/ColumnSelector/ Methods fit(X, y=None) Mock method. Does nothing. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] (default: None) Returns self fit_transform(X, y=None) Return a slice of the input array. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] (default: None) Returns X_slice : shape = [n_samples, k_features] Subset of the feature space where k_features <= n_features get_params(deep=True) Get parameters for this estimator. Parameters deep : boolean, optional If True, will return the parameters for this estimator and contained subobjects that are estimators. Returns params : mapping of string to any Parameter names mapped to their values. set_params( params) Set the parameters of this estimator. The method works on simple estimators as well as on nested objects (such as pipelines). The latter have parameters of the form <component>__<parameter> so that it's possible to update each component of a nested object. Returns self transform(X, y=None) Return a slice of the input array. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] (default: None) Returns X_slice : shape = [n_samples, k_features] Subset of the feature space where k_features <= n_features","title":"ColumnSelector: Scikit-learn utility function to select specific columns in a pipeline"},{"location":"user_guide/feature_selection/ColumnSelector/#columnselector-scikit-learn-utility-function-to-select-specific-columns-in-a-pipeline","text":"Implementation of a column selector class for scikit-learn pipelines. from mlxtend.feature_selection import ColumnSelector","title":"ColumnSelector: Scikit-learn utility function to select specific columns in a pipeline"},{"location":"user_guide/feature_selection/ColumnSelector/#overview","text":"The ColumnSelector can be used for \"manual\" feature selection, e.g., as part of a grid search via a scikit-learn pipeline.","title":"Overview"},{"location":"user_guide/feature_selection/ColumnSelector/#references","text":"-","title":"References"},{"location":"user_guide/feature_selection/ColumnSelector/#example-1-fitting-an-estimator-on-a-feature-subset","text":"Load a simple benchmark dataset: from sklearn.datasets import load_iris iris = load_iris() X = iris.data y = iris.target The ColumnSelector is a simple transformer class that selects specific columns (features) from a datast. For instance, using the transform method returns a reduced dataset that only contains two features (here: the first two features via the indices 0 and 1, respectively): from mlxtend.feature_selection import ColumnSelector col_selector = ColumnSelector(cols=(0, 1)) # col_selector.fit(X) # optional, does not do anything col_selector.transform(X).shape (150, 2) ColumnSelector works both with numpy arrays and pandas dataframes: import pandas as pd iris_df = pd.DataFrame(iris.data, columns=iris.feature_names) iris_df.head() .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } sepal length (cm) sepal width (cm) petal length (cm) petal width (cm) 0 5.1 3.5 1.4 0.2 1 4.9 3.0 1.4 0.2 2 4.7 3.2 1.3 0.2 3 4.6 3.1 1.5 0.2 4 5.0 3.6 1.4 0.2 col_selector = ColumnSelector(cols=(\"sepal length (cm)\", \"sepal width (cm)\")) col_selector.transform(iris_df).shape (150, 2) Similarly, we can use the ColumnSelector as part of a scikit-learn Pipeline : from sklearn.neighbors import KNeighborsClassifier from sklearn.preprocessing import StandardScaler from sklearn.pipeline import make_pipeline pipe = make_pipeline(StandardScaler(), ColumnSelector(cols=(0, 1)), KNeighborsClassifier()) pipe.fit(X, y) pipe.score(X, y) 0.84","title":"Example 1 - Fitting an Estimator on a Feature Subset"},{"location":"user_guide/feature_selection/ColumnSelector/#example-2-feature-selection-via-gridsearch","text":"Example 1 showed a simple useage example of the ColumnSelector ; however, selecting columns from a dataset is trivial and does not require a specific transformer class since we could have achieved the same results via classifier.fit(X[:, :2], y) classifier.score(X[:, :2], y) However, the ColumnSelector becomes really useful for feature selection as part of a grid search as shown in this example. Load a simple benchmark dataset: from sklearn.datasets import load_iris iris = load_iris() X = iris.data y = iris.target Create all possible combinations: from itertools import combinations all_comb = [] for size in range(1, 5): all_comb += list(combinations(range(X.shape[1]), r=size)) print(all_comb) [(0,), (1,), (2,), (3,), (0, 1), (0, 2), (0, 3), (1, 2), (1, 3), (2, 3), (0, 1, 2), (0, 1, 3), (0, 2, 3), (1, 2, 3), (0, 1, 2, 3)] Feature and model selection via grid search: from mlxtend.feature_selection import ColumnSelector from sklearn.neighbors import KNeighborsClassifier from sklearn.preprocessing import StandardScaler from sklearn.model_selection import GridSearchCV from sklearn.pipeline import make_pipeline pipe = make_pipeline(StandardScaler(), ColumnSelector(), KNeighborsClassifier()) param_grid = {'columnselector__cols': all_comb, 'kneighborsclassifier__n_neighbors': list(range(1, 11))} grid = GridSearchCV(pipe, param_grid, cv=5, n_jobs=-1) grid.fit(X, y) print('Best parameters:', grid.best_params_) print('Best performance:', grid.best_score_) Best parameters: {'columnselector__cols': (2, 3), 'kneighborsclassifier__n_neighbors': 1} Best performance: 0.98","title":"Example 2 - Feature Selection via GridSearch"},{"location":"user_guide/feature_selection/ColumnSelector/#example-3-scaling-of-a-subset-of-features-in-a-scikit-learn-pipeline","text":"The following example illustrates how we could use the ColumnSelector in tandem with scikit-learn's FeatureUnion to only scale certain features (in this toy example: the first and second feature only) in a datasets in a Pipeline . from mlxtend.feature_selection import ColumnSelector from sklearn.pipeline import make_pipeline from sklearn.pipeline import Pipeline from sklearn.pipeline import FeatureUnion from sklearn.preprocessing import MinMaxScaler from sklearn.neighbors import KNeighborsClassifier from mlxtend.data import iris_data X, y = iris_data() scale_pipe = make_pipeline(ColumnSelector(cols=(0, 1)), MinMaxScaler()) pipeline = Pipeline([ ('feats', FeatureUnion([ ('col_1-2', scale_pipe), ('col_3-4', ColumnSelector(cols=(2, 3))) ])), ('clf', KNeighborsClassifier()) ]) pipeline.fit(X, y) Pipeline(memory=None, steps=[('feats', FeatureUnion(n_jobs=None, transformer_list=[('col_1-2', Pipeline(memory=None, steps=[('columnselector', ColumnSelector(cols=(0, 1), drop_axis=False)), ('minmaxscaler', MinMaxScaler(copy=True, feature_range=(0, 1)))])), ('col_3-4', ColumnSelector(cols=(2, 3), drop_axis=Fa...ki', metric_params=None, n_jobs=None, n_neighbors=5, p=2, weights='uniform'))])","title":"Example 3 -- Scaling of a Subset of Features in a scikit-learn Pipeline"},{"location":"user_guide/feature_selection/ColumnSelector/#api","text":"ColumnSelector(cols=None, drop_axis=False) Object for selecting specific columns from a data set. Parameters cols : array-like (default: None) A list specifying the feature indices to be selected. For example, [1, 4, 5] to select the 2nd, 5th, and 6th feature columns. If None, returns all columns in the array. drop_axis : bool (default=False) Drops last axis if True and the only one column is selected. This is useful, e.g., when the ColumnSelector is used for selecting only one column and the resulting array should be fed to e.g., a scikit-learn column selector. E.g., instead of returning an array with shape (n_samples, 1), drop_axis=True will return an aray with shape (n_samples,). Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/feature_selection/ColumnSelector/","title":"API"},{"location":"user_guide/feature_selection/ColumnSelector/#methods","text":"fit(X, y=None) Mock method. Does nothing. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] (default: None) Returns self fit_transform(X, y=None) Return a slice of the input array. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] (default: None) Returns X_slice : shape = [n_samples, k_features] Subset of the feature space where k_features <= n_features get_params(deep=True) Get parameters for this estimator. Parameters deep : boolean, optional If True, will return the parameters for this estimator and contained subobjects that are estimators. Returns params : mapping of string to any Parameter names mapped to their values. set_params( params) Set the parameters of this estimator. The method works on simple estimators as well as on nested objects (such as pipelines). The latter have parameters of the form <component>__<parameter> so that it's possible to update each component of a nested object. Returns self transform(X, y=None) Return a slice of the input array. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] (default: None) Returns X_slice : shape = [n_samples, k_features] Subset of the feature space where k_features <= n_features","title":"Methods"},{"location":"user_guide/feature_selection/ExhaustiveFeatureSelector/","text":"ExhaustiveFeatureSelector: Optimal feature sets by considering all possible feature combinations Implementation of an exhaustive feature selector for sampling and evaluating all possible feature combinations in a specified range. from mlxtend.feature_selection import ExhaustiveFeatureSelector Overview This exhaustive feature selection algorithm is a wrapper approach for brute-force evaluation of feature subsets; the best subset is selected by optimizing a specified performance metric given an arbitrary regressor or classifier. For instance, if the classifier is a logistic regression and the dataset consists of 4 features, the alogorithm will evaluate all 15 feature combinations (if min_features=1 and max_features=4 ) {0} {1} {2} {3} {0, 1} {0, 2} {0, 3} {1, 2} {1, 3} {2, 3} {0, 1, 2} {0, 1, 3} {0, 2, 3} {1, 2, 3} {0, 1, 2, 3} and select the one that results in the best performance (e.g., classification accuracy) of the logistic regression classifier. Example 1 - A simple Iris example Initializing a simple classifier from scikit-learn: from sklearn.neighbors import KNeighborsClassifier from sklearn.datasets import load_iris from mlxtend.feature_selection import ExhaustiveFeatureSelector as EFS iris = load_iris() X = iris.data y = iris.target knn = KNeighborsClassifier(n_neighbors=3) efs1 = EFS(knn, min_features=1, max_features=4, scoring='accuracy', print_progress=True, cv=5) efs1 = efs1.fit(X, y) print('Best accuracy score: %.2f' % efs1.best_score_) print('Best subset (indices):', efs1.best_idx_) print('Best subset (corresponding names):', efs1.best_feature_names_) Features: 15/15 Best accuracy score: 0.97 Best subset (indices): (0, 2, 3) Best subset (corresponding names): ('0', '2', '3') Feature Names When working with large datasets, the feature indices might be hard to interpret. In this case, we recommend using pandas DataFrames with distinct column names as input: import pandas as pd df_X = pd.DataFrame(X, columns=[\"Sepal length\", \"Sepal width\", \"Petal length\", \"Petal width\"]) df_X.head() .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } Sepal length Sepal width Petal length Petal width 0 5.1 3.5 1.4 0.2 1 4.9 3.0 1.4 0.2 2 4.7 3.2 1.3 0.2 3 4.6 3.1 1.5 0.2 4 5.0 3.6 1.4 0.2 efs1 = efs1.fit(df_X, y) print('Best accuracy score: %.2f' % efs1.best_score_) print('Best subset (indices):', efs1.best_idx_) print('Best subset (corresponding names):', efs1.best_feature_names_) Features: 15/15 Best accuracy score: 0.97 Best subset (indices): (0, 2, 3) Best subset (corresponding names): ('Sepal length', 'Petal length', 'Petal width') Detailed Outputs Via the subsets_ attribute, we can take a look at the selected feature indices at each step: efs1.subsets_ {0: {'feature_idx': (0,), 'cv_scores': array([0.53333333, 0.63333333, 0.7 , 0.8 , 0.56666667]), 'avg_score': 0.6466666666666667, 'feature_names': ('Sepal length',)}, 1: {'feature_idx': (1,), 'cv_scores': array([0.43333333, 0.63333333, 0.53333333, 0.43333333, 0.5 ]), 'avg_score': 0.5066666666666666, 'feature_names': ('Sepal width',)}, 2: {'feature_idx': (2,), 'cv_scores': array([0.93333333, 0.93333333, 0.9 , 0.93333333, 1. ]), 'avg_score': 0.9400000000000001, 'feature_names': ('Petal length',)}, 3: {'feature_idx': (3,), 'cv_scores': array([0.96666667, 0.96666667, 0.93333333, 0.93333333, 1. ]), 'avg_score': 0.96, 'feature_names': ('Petal width',)}, 4: {'feature_idx': (0, 1), 'cv_scores': array([0.66666667, 0.8 , 0.7 , 0.86666667, 0.66666667]), 'avg_score': 0.74, 'feature_names': ('Sepal length', 'Sepal width')}, 5: {'feature_idx': (0, 2), 'cv_scores': array([0.96666667, 1. , 0.86666667, 0.93333333, 0.96666667]), 'avg_score': 0.9466666666666667, 'feature_names': ('Sepal length', 'Petal length')}, 6: {'feature_idx': (0, 3), 'cv_scores': array([0.96666667, 0.96666667, 0.9 , 0.93333333, 1. ]), 'avg_score': 0.9533333333333334, 'feature_names': ('Sepal length', 'Petal width')}, 7: {'feature_idx': (1, 2), 'cv_scores': array([0.93333333, 0.93333333, 0.9 , 0.93333333, 0.93333333]), 'avg_score': 0.9266666666666667, 'feature_names': ('Sepal width', 'Petal length')}, 8: {'feature_idx': (1, 3), 'cv_scores': array([0.96666667, 0.96666667, 0.86666667, 0.93333333, 0.96666667]), 'avg_score': 0.9400000000000001, 'feature_names': ('Sepal width', 'Petal width')}, 9: {'feature_idx': (2, 3), 'cv_scores': array([0.96666667, 0.96666667, 0.9 , 0.93333333, 1. ]), 'avg_score': 0.9533333333333334, 'feature_names': ('Petal length', 'Petal width')}, 10: {'feature_idx': (0, 1, 2), 'cv_scores': array([0.96666667, 0.96666667, 0.86666667, 0.93333333, 0.96666667]), 'avg_score': 0.9400000000000001, 'feature_names': ('Sepal length', 'Sepal width', 'Petal length')}, 11: {'feature_idx': (0, 1, 3), 'cv_scores': array([0.93333333, 0.96666667, 0.9 , 0.93333333, 1. ]), 'avg_score': 0.9466666666666667, 'feature_names': ('Sepal length', 'Sepal width', 'Petal width')}, 12: {'feature_idx': (0, 2, 3), 'cv_scores': array([0.96666667, 0.96666667, 0.96666667, 0.96666667, 1. ]), 'avg_score': 0.9733333333333334, 'feature_names': ('Sepal length', 'Petal length', 'Petal width')}, 13: {'feature_idx': (1, 2, 3), 'cv_scores': array([0.96666667, 0.96666667, 0.93333333, 0.93333333, 1. ]), 'avg_score': 0.96, 'feature_names': ('Sepal width', 'Petal length', 'Petal width')}, 14: {'feature_idx': (0, 1, 2, 3), 'cv_scores': array([0.96666667, 0.96666667, 0.93333333, 0.96666667, 1. ]), 'avg_score': 0.9666666666666668, 'feature_names': ('Sepal length', 'Sepal width', 'Petal length', 'Petal width')}} Example 2 - Visualizing the feature selection results For our convenience, we can visualize the output from the feature selection in a pandas DataFrame format using the get_metric_dict method of the ExhaustiveFeatureSelector object. The columns std_dev and std_err represent the standard deviation and standard errors of the cross-validation scores, respectively. Below, we see the DataFrame of the Sequential Forward Selector from Example 2: import pandas as pd iris = load_iris() X = iris.data y = iris.target knn = KNeighborsClassifier(n_neighbors=3) efs1 = EFS(knn, min_features=1, max_features=4, scoring='accuracy', print_progress=True, cv=5) feature_names = ('sepal length', 'sepal width', 'petal length', 'petal width') df_X = pd.DataFrame( X, columns=[\"Sepal length\", \"Sepal width\", \"Petal length\", \"Petal width\"]) efs1 = efs1.fit(df_X, y) df = pd.DataFrame.from_dict(efs1.get_metric_dict()).T df.sort_values('avg_score', inplace=True, ascending=False) df Features: 15/15 .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } feature_idx cv_scores avg_score feature_names ci_bound std_dev std_err 12 (0, 2, 3) [0.9666666666666667, 0.9666666666666667, 0.966... 0.973333 (Sepal length, Petal length, Petal width) 0.017137 0.013333 0.006667 14 (0, 1, 2, 3) [0.9666666666666667, 0.9666666666666667, 0.933... 0.966667 (Sepal length, Sepal width, Petal length, Peta... 0.027096 0.021082 0.010541 3 (3,) [0.9666666666666667, 0.9666666666666667, 0.933... 0.96 (Petal width,) 0.032061 0.024944 0.012472 13 (1, 2, 3) [0.9666666666666667, 0.9666666666666667, 0.933... 0.96 (Sepal width, Petal length, Petal width) 0.032061 0.024944 0.012472 6 (0, 3) [0.9666666666666667, 0.9666666666666667, 0.9, ... 0.953333 (Sepal length, Petal width) 0.043691 0.033993 0.016997 9 (2, 3) [0.9666666666666667, 0.9666666666666667, 0.9, ... 0.953333 (Petal length, Petal width) 0.043691 0.033993 0.016997 5 (0, 2) [0.9666666666666667, 1.0, 0.8666666666666667, ... 0.946667 (Sepal length, Petal length) 0.058115 0.045216 0.022608 11 (0, 1, 3) [0.9333333333333333, 0.9666666666666667, 0.9, ... 0.946667 (Sepal length, Sepal width, Petal width) 0.043691 0.033993 0.016997 2 (2,) [0.9333333333333333, 0.9333333333333333, 0.9, ... 0.94 (Petal length,) 0.041977 0.03266 0.01633 8 (1, 3) [0.9666666666666667, 0.9666666666666667, 0.866... 0.94 (Sepal width, Petal width) 0.049963 0.038873 0.019437 10 (0, 1, 2) [0.9666666666666667, 0.9666666666666667, 0.866... 0.94 (Sepal length, Sepal width, Petal length) 0.049963 0.038873 0.019437 7 (1, 2) [0.9333333333333333, 0.9333333333333333, 0.9, ... 0.926667 (Sepal width, Petal length) 0.017137 0.013333 0.006667 4 (0, 1) [0.6666666666666666, 0.8, 0.7, 0.8666666666666... 0.74 (Sepal length, Sepal width) 0.102823 0.08 0.04 0 (0,) [0.5333333333333333, 0.6333333333333333, 0.7, ... 0.646667 (Sepal length,) 0.122983 0.095685 0.047842 1 (1,) [0.43333333333333335, 0.6333333333333333, 0.53... 0.506667 (Sepal width,) 0.095416 0.074237 0.037118 import matplotlib.pyplot as plt metric_dict = efs1.get_metric_dict() fig = plt.figure() k_feat = sorted(metric_dict.keys()) avg = [metric_dict[k]['avg_score'] for k in k_feat] upper, lower = [], [] for k in k_feat: upper.append(metric_dict[k]['avg_score'] + metric_dict[k]['std_dev']) lower.append(metric_dict[k]['avg_score'] - metric_dict[k]['std_dev']) plt.fill_between(k_feat, upper, lower, alpha=0.2, color='blue', lw=1) plt.plot(k_feat, avg, color='blue', marker='o') plt.ylabel('Accuracy +/- Standard Deviation') plt.xlabel('Number of Features') feature_min = len(metric_dict[k_feat[0]]['feature_idx']) feature_max = len(metric_dict[k_feat[-1]]['feature_idx']) plt.xticks(k_feat, [str(metric_dict[k]['feature_names']) for k in k_feat], rotation=90) plt.show() Example 3 - Exhaustive feature selection for regression analysis Similar to the classification examples above, the SequentialFeatureSelector also supports scikit-learn's estimators for regression. from sklearn.linear_model import LinearRegression from sklearn.datasets import load_boston boston = load_boston() X, y = boston.data, boston.target lr = LinearRegression() efs = EFS(lr, min_features=10, max_features=12, scoring='neg_mean_squared_error', cv=10) efs.fit(X, y) print('Best MSE score: %.2f' % efs.best_score_ * (-1)) print('Best subset:', efs.best_idx_) /Users/sebastianraschka/miniforge3/lib/python3.9/site-packages/sklearn/utils/deprecation.py:87: FutureWarning: Function load_boston is deprecated; `load_boston` is deprecated in 1.0 and will be removed in 1.2. The Boston housing prices dataset has an ethical problem. You can refer to the documentation of this function for further details. The scikit-learn maintainers therefore strongly discourage the use of this dataset unless the purpose of the code is to study and educate about ethical issues in data science and machine learning. In this special case, you can fetch the dataset from the original source:: import pandas as pd import numpy as np data_url = \"https://lib.stat.cmu.edu/datasets/boston\" raw_df = pd.read_csv(data_url, sep=\"\\s+\", skiprows=22, header=None) data = np.hstack([raw_df.values[::2, :], raw_df.values[1::2, :2]]) target = raw_df.values[1::2, 2] Alternative datasets include the California housing dataset (i.e. :func:`~sklearn.datasets.fetch_california_housing`) and the Ames housing dataset. You can load the datasets as follows:: from sklearn.datasets import fetch_california_housing housing = fetch_california_housing() for the California housing dataset and:: from sklearn.datasets import fetch_openml housing = fetch_openml(name=\"house_prices\", as_frame=True) for the Ames housing dataset. warnings.warn(msg, category=FutureWarning) Features: 377/377 Best subset: (0, 1, 4, 6, 7, 8, 9, 10, 11, 12) Example 4 - Regression and adjusted R2 As shown in Example 3, the exhaustive feature selector can be used for selecting features via a regression model. In regression analysis, there exists the common phenomenon that the R^2 score can become spuriously inflated the more features we choose. Hence, and this is especially true for feature selection, it is useful to make model comparisons based on the adjusted R^2 value rather than the regular R^2 . The adjusted R^2 , \\bar{R}^{2} , accounts for the number of features and examples as follows: \\bar{R}^{2}=1-\\left(1-R^{2}\\right) \\frac{n-1}{n-p-1}, where n is the number of examples and p is the number of features. One of the advantages of scikit-learn's API is that it's consistent, intuitive, and simple to use. However, one downside of this API design is that it can be a bit restrictive for certain scenarios. For instance, scikit-learn scoring function only take two inputs, the predicted and the true target values. Hence, we cannot use scikit-learn's scoring API to compute the adjusted R^2 , which also requires the number of features. However, as a workaround, we can compute the R^2 for the different feature subsets and then do a posthoc computation to obtain the adjusted R^2 . Step 1: Compute R^2 : from sklearn.linear_model import LinearRegression from sklearn.datasets import load_boston boston = load_boston() X, y = boston.data, boston.target lr = LinearRegression() efs = EFS(lr, min_features=10, max_features=12, scoring='r2', cv=10) efs.fit(X, y) print('Best R2 score: %.2f' % efs.best_score_ * (-1)) print('Best subset:', efs.best_idx_) /Users/sebastianraschka/miniforge3/lib/python3.9/site-packages/sklearn/utils/deprecation.py:87: FutureWarning: Function load_boston is deprecated; `load_boston` is deprecated in 1.0 and will be removed in 1.2. The Boston housing prices dataset has an ethical problem. You can refer to the documentation of this function for further details. The scikit-learn maintainers therefore strongly discourage the use of this dataset unless the purpose of the code is to study and educate about ethical issues in data science and machine learning. In this special case, you can fetch the dataset from the original source:: import pandas as pd import numpy as np data_url = \"https://lib.stat.cmu.edu/datasets/boston\" raw_df = pd.read_csv(data_url, sep=\"\\s+\", skiprows=22, header=None) data = np.hstack([raw_df.values[::2, :], raw_df.values[1::2, :2]]) target = raw_df.values[1::2, 2] Alternative datasets include the California housing dataset (i.e. :func:`~sklearn.datasets.fetch_california_housing`) and the Ames housing dataset. You can load the datasets as follows:: from sklearn.datasets import fetch_california_housing housing = fetch_california_housing() for the California housing dataset and:: from sklearn.datasets import fetch_openml housing = fetch_openml(name=\"house_prices\", as_frame=True) for the Ames housing dataset. warnings.warn(msg, category=FutureWarning) Features: 377/377 Best subset: (1, 3, 5, 6, 7, 8, 9, 10, 11, 12) Step 2: Compute adjusted R^2 : def adjust_r2(r2, num_examples, num_features): coef = (num_examples - 1) / (num_examples - num_features - 1) return 1 - (1 - r2) * coef for i in efs.subsets_: efs.subsets_[i]['adjusted_avg_score'] = ( adjust_r2(r2=efs.subsets_[i]['avg_score'], num_examples=X.shape[0]/10, num_features=len(efs.subsets_[i]['feature_idx'])) ) Step 3: Select best subset based on adjusted R^2 : score = -99e10 for i in efs.subsets_: score = efs.subsets_[i]['adjusted_avg_score'] if ( efs.subsets_[i]['adjusted_avg_score'] == score and len(efs.subsets_[i]['feature_idx']) < len(efs.best_idx_) )\\ or efs.subsets_[i]['adjusted_avg_score'] > score: efs.best_idx_ = efs.subsets_[i]['feature_idx'] print('Best adjusted R2 score: %.2f' % efs.best_score_ * (-1)) print('Best subset:', efs.best_idx_) Best subset: (1, 3, 5, 6, 7, 8, 9, 10, 11, 12) Example 5 - Using the selected feature subset For making new predictions # Initialize the dataset from sklearn.neighbors import KNeighborsClassifier from sklearn.datasets import load_iris from sklearn.model_selection import train_test_split iris = load_iris() X, y = iris.data, iris.target X_train, X_test, y_train, y_test = train_test_split( X, y, test_size=0.33, random_state=1) knn = KNeighborsClassifier(n_neighbors=3) # Select the \"best\" three features via # 5-fold cross-validation on the training set. from mlxtend.feature_selection import ExhaustiveFeatureSelector as EFS efs1 = EFS(knn, min_features=1, max_features=4, scoring='accuracy', cv=5) efs1 = efs1.fit(X_train, y_train) Features: 15/15 print('Selected features:', efs1.best_idx_) Selected features: (2, 3) # Generate the new subsets based on the selected features # Note that the transform call is equivalent to # X_train[:, efs1.k_feature_idx_] X_train_efs = efs1.transform(X_train) X_test_efs = efs1.transform(X_test) # Fit the estimator using the new feature subset # and make a prediction on the test data knn.fit(X_train_efs, y_train) y_pred = knn.predict(X_test_efs) # Compute the accuracy of the prediction acc = float((y_test == y_pred).sum()) / y_pred.shape[0] print('Test set accuracy: %.2f %%' % (acc*100)) Test set accuracy: 96.00 % Example 6 - Exhaustive feature selection and GridSearch # Initialize the dataset from sklearn.datasets import load_iris from sklearn.model_selection import train_test_split iris = load_iris() X, y = iris.data, iris.target X_train, X_test, y_train, y_test = train_test_split( X, y, test_size=0.33, random_state=1) Use scikit-learn's GridSearch to tune the hyperparameters of the LogisticRegression estimator inside the ExhaustiveFeatureSelector and use it for prediction in the pipeline. Note that the clone_estimator attribute needs to be set to False . from sklearn.model_selection import GridSearchCV from sklearn.pipeline import make_pipeline from sklearn.linear_model import LogisticRegression from mlxtend.feature_selection import ExhaustiveFeatureSelector as EFS lr = LogisticRegression(multi_class='multinomial', solver='newton-cg', random_state=123) efs1 = EFS(estimator=lr, min_features=2, max_features=3, scoring='accuracy', print_progress=False, clone_estimator=False, cv=5, n_jobs=1) pipe = make_pipeline(efs1, lr) param_grid = {'exhaustivefeatureselector__estimator__C': [0.1, 1.0, 10.0]} gs = GridSearchCV(estimator=pipe, param_grid=param_grid, scoring='accuracy', n_jobs=1, cv=2, verbose=1, refit=False) # run gridearch gs = gs.fit(X_train, y_train) Fitting 2 folds for each of 3 candidates, totalling 6 fits ... and the \"best\" parameters determined by GridSearch are ... print(\"Best parameters via GridSearch\", gs.best_params_) Best parameters via GridSearch {'exhaustivefeatureselector__estimator__C': 0.1} Obtaining the best k feature indices after GridSearch If we are interested in the best k best feature indices via SequentialFeatureSelection.best_idx_ , we have to initialize a GridSearchCV object with refit=True . Now, the grid search object will take the complete training dataset and the best parameters, which it found via cross-validation, to train the estimator pipeline. gs = GridSearchCV(estimator=pipe, param_grid=param_grid, scoring='accuracy', n_jobs=1, cv=2, verbose=1, refit=True) After running the grid search, we can access the individual pipeline objects of the best_estimator_ via the steps attribute. gs = gs.fit(X_train, y_train) gs.best_estimator_.steps Fitting 2 folds for each of 3 candidates, totalling 6 fits [('exhaustivefeatureselector', ExhaustiveFeatureSelector(clone_estimator=False, estimator=LogisticRegression(C=0.1, multi_class='multinomial', random_state=123, solver='newton-cg'), feature_groups=[[0], [1], [2], [3]], max_features=3, min_features=2, print_progress=False)), ('logisticregression', LogisticRegression(multi_class='multinomial', random_state=123, solver='newton-cg'))] Via sub-indexing, we can then obtain the best-selected feature subset: print('Best features:', gs.best_estimator_.steps[0][1].best_idx_) Best features: (2, 3) During cross-validation, this feature combination had a CV accuracy of: print('Best score:', gs.best_score_) Best score: 0.96 gs.best_params_ {'exhaustivefeatureselector__estimator__C': 0.1} Alternatively , if we can set the \"best grid search parameters\" in our pipeline manually if we ran GridSearchCV with refit=False . It should yield the same results: pipe.set_params(**gs.best_params_).fit(X_train, y_train) print('Best features:', pipe.steps[0][1].best_idx_) Best features: (2, 3) Example 7 - Exhaustive Feature Selection with LOOCV The ExhaustiveFeatureSelector is not restricted to k-fold cross-validation. You can use any type of cross-validation method that supports the general scikit-learn cross-validation API. The following example illustrates the use of scikit-learn's LeaveOneOut cross-validation method in combination with the exhaustive feature selector. from sklearn.neighbors import KNeighborsClassifier from sklearn.datasets import load_iris from mlxtend.feature_selection import ExhaustiveFeatureSelector as EFS from sklearn.model_selection import LeaveOneOut iris = load_iris() X = iris.data y = iris.target knn = KNeighborsClassifier(n_neighbors=3) efs1 = EFS(knn, min_features=1, max_features=4, scoring='accuracy', print_progress=True, cv=LeaveOneOut()) ### Use cross-validation generator here efs1 = efs1.fit(X, y) print('Best accuracy score: %.2f' % efs1.best_score_) print('Best subset (indices):', efs1.best_idx_) print('Best subset (corresponding names):', efs1.best_feature_names_) Features: 15/15 Best accuracy score: 0.96 Best subset (indices): (3,) Best subset (corresponding names): ('3',) Example 8 - Interrupting Long Runs for Intermediate Results If your run is taking too long, it is possible to trigger a KeyboardInterrupt (e.g., ctrl+c on a Mac, or interrupting the cell in a Jupyter notebook) to obtain temporary results. Toy dataset from sklearn.datasets import make_classification from sklearn.model_selection import train_test_split X, y = make_classification( n_samples=200000, n_features=6, n_informative=2, n_redundant=1, n_repeated=1, n_clusters_per_class=2, flip_y=0.05, class_sep=0.5, random_state=123, ) X_train, X_test, y_train, y_test = train_test_split( X, y, test_size=0.2, random_state=123 ) Long run with interruption from mlxtend.feature_selection import ExhaustiveFeatureSelector as EFS from sklearn.linear_model import LogisticRegression model = LogisticRegression(max_iter=10000) efs1 = EFS(model, min_features=1, max_features=4, print_progress=True, scoring='accuracy') efs1 = efs1.fit(X_train, y_train) Features: 56/56 Finalizing the fit Note that the feature selection run hasn't finished, so certain attributes may not be available. In order to use the EFS instance, it is recommended to call finalize_fit , which will make EFS estimator appear as \"fitted\" process the temporary results: efs1.finalize_fit() print('Best accuracy score: %.2f' % efs1.best_score_) print('Best subset (indices):', efs1.best_idx_) Best accuracy score: 0.73 Best subset (indices): (1, 2) Example 9 - Working with Feature Groups Since mlxtend v0.21.0, it is possible to specify feature groups. Feature groups allow you to group certain features together, such that they are always selected as a group. This can be very useful in contexts similar to one-hot encoding -- if you want to treat the one-hot encoded feature as a single feature: In the following example, we specify sepal length and sepal width as a feature group so that they are always selected together: from sklearn.datasets import load_iris import pandas as pd iris = load_iris() X = iris.data y = iris.target X_df = pd.DataFrame(X, columns=['sepal len', 'petal len', 'sepal wid', 'petal wid']) X_df.head() .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } sepal len petal len sepal wid petal wid 0 5.1 3.5 1.4 0.2 1 4.9 3.0 1.4 0.2 2 4.7 3.2 1.3 0.2 3 4.6 3.1 1.5 0.2 4 5.0 3.6 1.4 0.2 from sklearn.neighbors import KNeighborsClassifier from mlxtend.feature_selection import ExhaustiveFeatureSelector as EFS knn = KNeighborsClassifier(n_neighbors=3) efs1 = EFS(knn, min_features=2, max_features=2, scoring='accuracy', feature_groups=[['sepal len', 'sepal wid'], ['petal len'], ['petal wid']], cv=3) efs1 = efs1.fit(X_df, y) print('Best accuracy score: %.2f' % efs1.best_score_) print('Best subset (indices):', efs1.best_idx_) print('Best subset (corresponding names):', efs1.best_feature_names_) Features: 3/3 Best accuracy score: 0.97 Best subset (indices): (0, 2, 3) Best subset (corresponding names): ('sepal len', 'sepal wid', 'petal wid') Notice that the returned number of features is 3, since the number of min_features and max_features corresponds to the number of feature groups. I.e., we have 2 feature groups in ['sepal len', 'sepal wid'], ['petal wid'] , but it expands to 3 features. API ExhaustiveFeatureSelector(estimator, min_features=1, max_features=1, print_progress=True, scoring='accuracy', cv=5, n_jobs=1, pre_dispatch='2 n_jobs', clone_estimator=True, fixed_features=None, feature_groups=None)* Exhaustive Feature Selection for Classification and Regression. (new in v0.4.3) Parameters estimator : scikit-learn classifier or regressor min_features : int (default: 1) Minumum number of features to select max_features : int (default: 1) Maximum number of features to select. If parameter feature_groups is not None, the number of features is equal to the number of feature groups, i.e. len(feature_groups) . For example, if feature_groups = [[0], [1], [2, 3], [4]] , then the max_features value cannot exceed 4. print_progress : bool (default: True) Prints progress as the number of epochs to stderr. scoring : str, (default='accuracy') Scoring metric in {accuracy, f1, precision, recall, roc_auc} for classifiers, {'mean_absolute_error', 'mean_squared_error', 'median_absolute_error', 'r2'} for regressors, or a callable object or function with signature scorer(estimator, X, y) . cv : int (default: 5) Scikit-learn cross-validation generator or int . If estimator is a classifier (or y consists of integer class labels), stratified k-fold is performed, and regular k-fold cross-validation otherwise. No cross-validation if cv is None, False, or 0. n_jobs : int (default: 1) The number of CPUs to use for evaluating different feature subsets in parallel. -1 means 'all CPUs'. pre_dispatch : int, or string (default: '2*n_jobs') Controls the number of jobs that get dispatched during parallel execution if n_jobs > 1 or n_jobs=-1 . Reducing this number can be useful to avoid an explosion of memory consumption when more jobs get dispatched than CPUs can process. This parameter can be: None, in which case all the jobs are immediately created and spawned. Use this for lightweight and fast-running jobs, to avoid delays due to on-demand spawning of the jobs An int, giving the exact number of total jobs that are spawned A string, giving an expression as a function of n_jobs, as in 2*n_jobs clone_estimator : bool (default: True) Clones estimator if True; works with the original estimator instance if False. Set to False if the estimator doesn't implement scikit-learn's set_params and get_params methods. In addition, it is required to set cv=0, and n_jobs=1. fixed_features : tuple (default: None) If not None , the feature indices provided as a tuple will be regarded as fixed by the feature selector. For example, if fixed_features=(1, 3, 7) , the 2nd, 4th, and 8th feature are guaranteed to be present in the solution. Note that if fixed_features is not None , make sure that the number of features to be selected is greater than len(fixed_features) . In other words, ensure that k_features > len(fixed_features) . feature_groups : list or None (default: None) Optional argument for treating certain features as a group. This means, the features within a group are always selected together, never split. For example, feature_groups=[[1], [2], [3, 4, 5]] specifies 3 feature groups.In this case, possible feature selection results with k_features=2 are [[1], [2] , [[1], [3, 4, 5]] , or [[2], [3, 4, 5]] . Feature groups can be useful for interpretability, for example, if features 3, 4, 5 are one-hot encoded features. (For more details, please read the notes at the bottom of this docstring). New in mlxtend v. 0.21.0. Attributes best_idx_ : array-like, shape = [n_predictions] Feature Indices of the selected feature subsets. best_feature_names_ : array-like, shape = [n_predictions] Feature names of the selected feature subsets. If pandas DataFrames are used in the fit method, the feature names correspond to the column names. Otherwise, the feature names are string representation of the feature array indices. New in v 0.13.0. best_score_ : float Cross validation average score of the selected subset. subsets_ : dict A dictionary of selected feature subsets during the exhaustive selection, where the dictionary keys are the lengths k of these feature subsets. The dictionary values are dictionaries themselves with the following keys: 'feature_idx' (tuple of indices of the feature subset) 'feature_names' (tuple of feature names of the feat. subset) 'cv_scores' (list individual cross-validation scores) 'avg_score' (average cross-validation score) Note that if pandas DataFrames are used in the fit method, the 'feature_names' correspond to the column names. Otherwise, the feature names are string representation of the feature array indices. The 'feature_names' is new in v. 0.13.0. Notes (1) If parameter feature_groups is not None, the number of features is equal to the number of feature groups, i.e. len(feature_groups) . For example, if feature_groups = [[0], [1], [2, 3], [4]] , then the max_features value cannot exceed 4. (2) Although two or more individual features may be considered as one group throughout the feature-selection process, it does not mean the individual features of that group have the same impact on the outcome. For instance, in linear regression, the coefficient of the feature 2 and 3 can be different even if they are considered as one group in feature_groups. (3) If both fixed_features and feature_groups are specified, ensure that each feature group contains the fixed_features selection. E.g., for a 3-feature set fixed_features=[0, 1] and feature_groups=[[0, 1], [2]] is valid; fixed_features=[0, 1] and feature_groups=[[0], [1, 2]] is not valid. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/feature_selection/ExhaustiveFeatureSelector/ Methods fit(X, y, groups=None, fit_params) Perform feature selection and learn model from training data. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. New in v 0.13.0: pandas DataFrames are now also accepted as argument for X. y : array-like, shape = [n_samples] Target values. groups : array-like, with shape (n_samples,), optional Group labels for the samples used while splitting the dataset into train/test set. Passed to the fit method of the cross-validator. fit_params : dict of string -> object, optional Parameters to pass to to the fit method of classifier. Returns self : object fit_transform(X, y, groups=None, fit_params) Fit to training data and return the best selected features from X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. New in v 0.13.0: pandas DataFrames are now also accepted as argument for X. y : array-like, shape = [n_samples] Target values. groups : array-like, with shape (n_samples,), optional Group labels for the samples used while splitting the dataset into train/test set. Passed to the fit method of the cross-validator. fit_params : dict of string -> object, optional Parameters to pass to to the fit method of classifier. Returns Feature subset of X, shape={n_samples, k_features} get_metric_dict(confidence_interval=0.95) Return metric dictionary Parameters confidence_interval : float (default: 0.95) A positive float between 0.0 and 1.0 to compute the confidence interval bounds of the CV score averages. Returns Dictionary with items where each dictionary value is a list with the number of iterations (number of feature subsets) as its length. The dictionary keys corresponding to these lists are as follows: 'feature_idx': tuple of the indices of the feature subset 'cv_scores': list with individual CV scores 'avg_score': of CV average scores 'std_dev': standard deviation of the CV score average 'std_err': standard error of the CV score average 'ci_bound': confidence interval bound of the CV score average get_params(deep=True) Get parameters for this estimator. Parameters deep : bool, default=True If True, will return the parameters for this estimator and contained subobjects that are estimators. Returns params : dict Parameter names mapped to their values. set_params( params) Set the parameters of this estimator. The method works on simple estimators as well as on nested objects (such as :class:`~sklearn.pipeline.Pipeline`). The latter have parameters of the form ``<component>__<parameter>`` so that it's possible to update each component of a nested object. Parameters **params : dict Estimator parameters. Returns self : estimator instance Estimator instance. transform(X) Return the best selected features from X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. New in v 0.13.0: pandas DataFrames are now also accepted as argument for X. Returns Feature subset of X, shape={n_samples, k_features}","title":"ExhaustiveFeatureSelector: Optimal feature sets by considering all possible feature combinations"},{"location":"user_guide/feature_selection/ExhaustiveFeatureSelector/#exhaustivefeatureselector-optimal-feature-sets-by-considering-all-possible-feature-combinations","text":"Implementation of an exhaustive feature selector for sampling and evaluating all possible feature combinations in a specified range. from mlxtend.feature_selection import ExhaustiveFeatureSelector","title":"ExhaustiveFeatureSelector: Optimal feature sets by considering all possible feature combinations"},{"location":"user_guide/feature_selection/ExhaustiveFeatureSelector/#overview","text":"This exhaustive feature selection algorithm is a wrapper approach for brute-force evaluation of feature subsets; the best subset is selected by optimizing a specified performance metric given an arbitrary regressor or classifier. For instance, if the classifier is a logistic regression and the dataset consists of 4 features, the alogorithm will evaluate all 15 feature combinations (if min_features=1 and max_features=4 ) {0} {1} {2} {3} {0, 1} {0, 2} {0, 3} {1, 2} {1, 3} {2, 3} {0, 1, 2} {0, 1, 3} {0, 2, 3} {1, 2, 3} {0, 1, 2, 3} and select the one that results in the best performance (e.g., classification accuracy) of the logistic regression classifier.","title":"Overview"},{"location":"user_guide/feature_selection/ExhaustiveFeatureSelector/#example-1-a-simple-iris-example","text":"Initializing a simple classifier from scikit-learn: from sklearn.neighbors import KNeighborsClassifier from sklearn.datasets import load_iris from mlxtend.feature_selection import ExhaustiveFeatureSelector as EFS iris = load_iris() X = iris.data y = iris.target knn = KNeighborsClassifier(n_neighbors=3) efs1 = EFS(knn, min_features=1, max_features=4, scoring='accuracy', print_progress=True, cv=5) efs1 = efs1.fit(X, y) print('Best accuracy score: %.2f' % efs1.best_score_) print('Best subset (indices):', efs1.best_idx_) print('Best subset (corresponding names):', efs1.best_feature_names_) Features: 15/15 Best accuracy score: 0.97 Best subset (indices): (0, 2, 3) Best subset (corresponding names): ('0', '2', '3')","title":"Example 1 - A simple Iris example"},{"location":"user_guide/feature_selection/ExhaustiveFeatureSelector/#feature-names","text":"When working with large datasets, the feature indices might be hard to interpret. In this case, we recommend using pandas DataFrames with distinct column names as input: import pandas as pd df_X = pd.DataFrame(X, columns=[\"Sepal length\", \"Sepal width\", \"Petal length\", \"Petal width\"]) df_X.head() .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } Sepal length Sepal width Petal length Petal width 0 5.1 3.5 1.4 0.2 1 4.9 3.0 1.4 0.2 2 4.7 3.2 1.3 0.2 3 4.6 3.1 1.5 0.2 4 5.0 3.6 1.4 0.2 efs1 = efs1.fit(df_X, y) print('Best accuracy score: %.2f' % efs1.best_score_) print('Best subset (indices):', efs1.best_idx_) print('Best subset (corresponding names):', efs1.best_feature_names_) Features: 15/15 Best accuracy score: 0.97 Best subset (indices): (0, 2, 3) Best subset (corresponding names): ('Sepal length', 'Petal length', 'Petal width')","title":"Feature Names"},{"location":"user_guide/feature_selection/ExhaustiveFeatureSelector/#detailed-outputs","text":"Via the subsets_ attribute, we can take a look at the selected feature indices at each step: efs1.subsets_ {0: {'feature_idx': (0,), 'cv_scores': array([0.53333333, 0.63333333, 0.7 , 0.8 , 0.56666667]), 'avg_score': 0.6466666666666667, 'feature_names': ('Sepal length',)}, 1: {'feature_idx': (1,), 'cv_scores': array([0.43333333, 0.63333333, 0.53333333, 0.43333333, 0.5 ]), 'avg_score': 0.5066666666666666, 'feature_names': ('Sepal width',)}, 2: {'feature_idx': (2,), 'cv_scores': array([0.93333333, 0.93333333, 0.9 , 0.93333333, 1. ]), 'avg_score': 0.9400000000000001, 'feature_names': ('Petal length',)}, 3: {'feature_idx': (3,), 'cv_scores': array([0.96666667, 0.96666667, 0.93333333, 0.93333333, 1. ]), 'avg_score': 0.96, 'feature_names': ('Petal width',)}, 4: {'feature_idx': (0, 1), 'cv_scores': array([0.66666667, 0.8 , 0.7 , 0.86666667, 0.66666667]), 'avg_score': 0.74, 'feature_names': ('Sepal length', 'Sepal width')}, 5: {'feature_idx': (0, 2), 'cv_scores': array([0.96666667, 1. , 0.86666667, 0.93333333, 0.96666667]), 'avg_score': 0.9466666666666667, 'feature_names': ('Sepal length', 'Petal length')}, 6: {'feature_idx': (0, 3), 'cv_scores': array([0.96666667, 0.96666667, 0.9 , 0.93333333, 1. ]), 'avg_score': 0.9533333333333334, 'feature_names': ('Sepal length', 'Petal width')}, 7: {'feature_idx': (1, 2), 'cv_scores': array([0.93333333, 0.93333333, 0.9 , 0.93333333, 0.93333333]), 'avg_score': 0.9266666666666667, 'feature_names': ('Sepal width', 'Petal length')}, 8: {'feature_idx': (1, 3), 'cv_scores': array([0.96666667, 0.96666667, 0.86666667, 0.93333333, 0.96666667]), 'avg_score': 0.9400000000000001, 'feature_names': ('Sepal width', 'Petal width')}, 9: {'feature_idx': (2, 3), 'cv_scores': array([0.96666667, 0.96666667, 0.9 , 0.93333333, 1. ]), 'avg_score': 0.9533333333333334, 'feature_names': ('Petal length', 'Petal width')}, 10: {'feature_idx': (0, 1, 2), 'cv_scores': array([0.96666667, 0.96666667, 0.86666667, 0.93333333, 0.96666667]), 'avg_score': 0.9400000000000001, 'feature_names': ('Sepal length', 'Sepal width', 'Petal length')}, 11: {'feature_idx': (0, 1, 3), 'cv_scores': array([0.93333333, 0.96666667, 0.9 , 0.93333333, 1. ]), 'avg_score': 0.9466666666666667, 'feature_names': ('Sepal length', 'Sepal width', 'Petal width')}, 12: {'feature_idx': (0, 2, 3), 'cv_scores': array([0.96666667, 0.96666667, 0.96666667, 0.96666667, 1. ]), 'avg_score': 0.9733333333333334, 'feature_names': ('Sepal length', 'Petal length', 'Petal width')}, 13: {'feature_idx': (1, 2, 3), 'cv_scores': array([0.96666667, 0.96666667, 0.93333333, 0.93333333, 1. ]), 'avg_score': 0.96, 'feature_names': ('Sepal width', 'Petal length', 'Petal width')}, 14: {'feature_idx': (0, 1, 2, 3), 'cv_scores': array([0.96666667, 0.96666667, 0.93333333, 0.96666667, 1. ]), 'avg_score': 0.9666666666666668, 'feature_names': ('Sepal length', 'Sepal width', 'Petal length', 'Petal width')}}","title":"Detailed Outputs"},{"location":"user_guide/feature_selection/ExhaustiveFeatureSelector/#example-2-visualizing-the-feature-selection-results","text":"For our convenience, we can visualize the output from the feature selection in a pandas DataFrame format using the get_metric_dict method of the ExhaustiveFeatureSelector object. The columns std_dev and std_err represent the standard deviation and standard errors of the cross-validation scores, respectively. Below, we see the DataFrame of the Sequential Forward Selector from Example 2: import pandas as pd iris = load_iris() X = iris.data y = iris.target knn = KNeighborsClassifier(n_neighbors=3) efs1 = EFS(knn, min_features=1, max_features=4, scoring='accuracy', print_progress=True, cv=5) feature_names = ('sepal length', 'sepal width', 'petal length', 'petal width') df_X = pd.DataFrame( X, columns=[\"Sepal length\", \"Sepal width\", \"Petal length\", \"Petal width\"]) efs1 = efs1.fit(df_X, y) df = pd.DataFrame.from_dict(efs1.get_metric_dict()).T df.sort_values('avg_score', inplace=True, ascending=False) df Features: 15/15 .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } feature_idx cv_scores avg_score feature_names ci_bound std_dev std_err 12 (0, 2, 3) [0.9666666666666667, 0.9666666666666667, 0.966... 0.973333 (Sepal length, Petal length, Petal width) 0.017137 0.013333 0.006667 14 (0, 1, 2, 3) [0.9666666666666667, 0.9666666666666667, 0.933... 0.966667 (Sepal length, Sepal width, Petal length, Peta... 0.027096 0.021082 0.010541 3 (3,) [0.9666666666666667, 0.9666666666666667, 0.933... 0.96 (Petal width,) 0.032061 0.024944 0.012472 13 (1, 2, 3) [0.9666666666666667, 0.9666666666666667, 0.933... 0.96 (Sepal width, Petal length, Petal width) 0.032061 0.024944 0.012472 6 (0, 3) [0.9666666666666667, 0.9666666666666667, 0.9, ... 0.953333 (Sepal length, Petal width) 0.043691 0.033993 0.016997 9 (2, 3) [0.9666666666666667, 0.9666666666666667, 0.9, ... 0.953333 (Petal length, Petal width) 0.043691 0.033993 0.016997 5 (0, 2) [0.9666666666666667, 1.0, 0.8666666666666667, ... 0.946667 (Sepal length, Petal length) 0.058115 0.045216 0.022608 11 (0, 1, 3) [0.9333333333333333, 0.9666666666666667, 0.9, ... 0.946667 (Sepal length, Sepal width, Petal width) 0.043691 0.033993 0.016997 2 (2,) [0.9333333333333333, 0.9333333333333333, 0.9, ... 0.94 (Petal length,) 0.041977 0.03266 0.01633 8 (1, 3) [0.9666666666666667, 0.9666666666666667, 0.866... 0.94 (Sepal width, Petal width) 0.049963 0.038873 0.019437 10 (0, 1, 2) [0.9666666666666667, 0.9666666666666667, 0.866... 0.94 (Sepal length, Sepal width, Petal length) 0.049963 0.038873 0.019437 7 (1, 2) [0.9333333333333333, 0.9333333333333333, 0.9, ... 0.926667 (Sepal width, Petal length) 0.017137 0.013333 0.006667 4 (0, 1) [0.6666666666666666, 0.8, 0.7, 0.8666666666666... 0.74 (Sepal length, Sepal width) 0.102823 0.08 0.04 0 (0,) [0.5333333333333333, 0.6333333333333333, 0.7, ... 0.646667 (Sepal length,) 0.122983 0.095685 0.047842 1 (1,) [0.43333333333333335, 0.6333333333333333, 0.53... 0.506667 (Sepal width,) 0.095416 0.074237 0.037118 import matplotlib.pyplot as plt metric_dict = efs1.get_metric_dict() fig = plt.figure() k_feat = sorted(metric_dict.keys()) avg = [metric_dict[k]['avg_score'] for k in k_feat] upper, lower = [], [] for k in k_feat: upper.append(metric_dict[k]['avg_score'] + metric_dict[k]['std_dev']) lower.append(metric_dict[k]['avg_score'] - metric_dict[k]['std_dev']) plt.fill_between(k_feat, upper, lower, alpha=0.2, color='blue', lw=1) plt.plot(k_feat, avg, color='blue', marker='o') plt.ylabel('Accuracy +/- Standard Deviation') plt.xlabel('Number of Features') feature_min = len(metric_dict[k_feat[0]]['feature_idx']) feature_max = len(metric_dict[k_feat[-1]]['feature_idx']) plt.xticks(k_feat, [str(metric_dict[k]['feature_names']) for k in k_feat], rotation=90) plt.show()","title":"Example 2 - Visualizing the feature selection results"},{"location":"user_guide/feature_selection/ExhaustiveFeatureSelector/#example-3-exhaustive-feature-selection-for-regression-analysis","text":"Similar to the classification examples above, the SequentialFeatureSelector also supports scikit-learn's estimators for regression. from sklearn.linear_model import LinearRegression from sklearn.datasets import load_boston boston = load_boston() X, y = boston.data, boston.target lr = LinearRegression() efs = EFS(lr, min_features=10, max_features=12, scoring='neg_mean_squared_error', cv=10) efs.fit(X, y) print('Best MSE score: %.2f' % efs.best_score_ * (-1)) print('Best subset:', efs.best_idx_) /Users/sebastianraschka/miniforge3/lib/python3.9/site-packages/sklearn/utils/deprecation.py:87: FutureWarning: Function load_boston is deprecated; `load_boston` is deprecated in 1.0 and will be removed in 1.2. The Boston housing prices dataset has an ethical problem. You can refer to the documentation of this function for further details. The scikit-learn maintainers therefore strongly discourage the use of this dataset unless the purpose of the code is to study and educate about ethical issues in data science and machine learning. In this special case, you can fetch the dataset from the original source:: import pandas as pd import numpy as np data_url = \"https://lib.stat.cmu.edu/datasets/boston\" raw_df = pd.read_csv(data_url, sep=\"\\s+\", skiprows=22, header=None) data = np.hstack([raw_df.values[::2, :], raw_df.values[1::2, :2]]) target = raw_df.values[1::2, 2] Alternative datasets include the California housing dataset (i.e. :func:`~sklearn.datasets.fetch_california_housing`) and the Ames housing dataset. You can load the datasets as follows:: from sklearn.datasets import fetch_california_housing housing = fetch_california_housing() for the California housing dataset and:: from sklearn.datasets import fetch_openml housing = fetch_openml(name=\"house_prices\", as_frame=True) for the Ames housing dataset. warnings.warn(msg, category=FutureWarning) Features: 377/377 Best subset: (0, 1, 4, 6, 7, 8, 9, 10, 11, 12)","title":"Example 3 - Exhaustive feature selection for regression analysis"},{"location":"user_guide/feature_selection/ExhaustiveFeatureSelector/#example-4-regression-and-adjusted-r2","text":"As shown in Example 3, the exhaustive feature selector can be used for selecting features via a regression model. In regression analysis, there exists the common phenomenon that the R^2 score can become spuriously inflated the more features we choose. Hence, and this is especially true for feature selection, it is useful to make model comparisons based on the adjusted R^2 value rather than the regular R^2 . The adjusted R^2 , \\bar{R}^{2} , accounts for the number of features and examples as follows: \\bar{R}^{2}=1-\\left(1-R^{2}\\right) \\frac{n-1}{n-p-1}, where n is the number of examples and p is the number of features. One of the advantages of scikit-learn's API is that it's consistent, intuitive, and simple to use. However, one downside of this API design is that it can be a bit restrictive for certain scenarios. For instance, scikit-learn scoring function only take two inputs, the predicted and the true target values. Hence, we cannot use scikit-learn's scoring API to compute the adjusted R^2 , which also requires the number of features. However, as a workaround, we can compute the R^2 for the different feature subsets and then do a posthoc computation to obtain the adjusted R^2 . Step 1: Compute R^2 : from sklearn.linear_model import LinearRegression from sklearn.datasets import load_boston boston = load_boston() X, y = boston.data, boston.target lr = LinearRegression() efs = EFS(lr, min_features=10, max_features=12, scoring='r2', cv=10) efs.fit(X, y) print('Best R2 score: %.2f' % efs.best_score_ * (-1)) print('Best subset:', efs.best_idx_) /Users/sebastianraschka/miniforge3/lib/python3.9/site-packages/sklearn/utils/deprecation.py:87: FutureWarning: Function load_boston is deprecated; `load_boston` is deprecated in 1.0 and will be removed in 1.2. The Boston housing prices dataset has an ethical problem. You can refer to the documentation of this function for further details. The scikit-learn maintainers therefore strongly discourage the use of this dataset unless the purpose of the code is to study and educate about ethical issues in data science and machine learning. In this special case, you can fetch the dataset from the original source:: import pandas as pd import numpy as np data_url = \"https://lib.stat.cmu.edu/datasets/boston\" raw_df = pd.read_csv(data_url, sep=\"\\s+\", skiprows=22, header=None) data = np.hstack([raw_df.values[::2, :], raw_df.values[1::2, :2]]) target = raw_df.values[1::2, 2] Alternative datasets include the California housing dataset (i.e. :func:`~sklearn.datasets.fetch_california_housing`) and the Ames housing dataset. You can load the datasets as follows:: from sklearn.datasets import fetch_california_housing housing = fetch_california_housing() for the California housing dataset and:: from sklearn.datasets import fetch_openml housing = fetch_openml(name=\"house_prices\", as_frame=True) for the Ames housing dataset. warnings.warn(msg, category=FutureWarning) Features: 377/377 Best subset: (1, 3, 5, 6, 7, 8, 9, 10, 11, 12) Step 2: Compute adjusted R^2 : def adjust_r2(r2, num_examples, num_features): coef = (num_examples - 1) / (num_examples - num_features - 1) return 1 - (1 - r2) * coef for i in efs.subsets_: efs.subsets_[i]['adjusted_avg_score'] = ( adjust_r2(r2=efs.subsets_[i]['avg_score'], num_examples=X.shape[0]/10, num_features=len(efs.subsets_[i]['feature_idx'])) ) Step 3: Select best subset based on adjusted R^2 : score = -99e10 for i in efs.subsets_: score = efs.subsets_[i]['adjusted_avg_score'] if ( efs.subsets_[i]['adjusted_avg_score'] == score and len(efs.subsets_[i]['feature_idx']) < len(efs.best_idx_) )\\ or efs.subsets_[i]['adjusted_avg_score'] > score: efs.best_idx_ = efs.subsets_[i]['feature_idx'] print('Best adjusted R2 score: %.2f' % efs.best_score_ * (-1)) print('Best subset:', efs.best_idx_) Best subset: (1, 3, 5, 6, 7, 8, 9, 10, 11, 12)","title":"Example 4 - Regression and adjusted R2"},{"location":"user_guide/feature_selection/ExhaustiveFeatureSelector/#example-5-using-the-selected-feature-subset-for-making-new-predictions","text":"# Initialize the dataset from sklearn.neighbors import KNeighborsClassifier from sklearn.datasets import load_iris from sklearn.model_selection import train_test_split iris = load_iris() X, y = iris.data, iris.target X_train, X_test, y_train, y_test = train_test_split( X, y, test_size=0.33, random_state=1) knn = KNeighborsClassifier(n_neighbors=3) # Select the \"best\" three features via # 5-fold cross-validation on the training set. from mlxtend.feature_selection import ExhaustiveFeatureSelector as EFS efs1 = EFS(knn, min_features=1, max_features=4, scoring='accuracy', cv=5) efs1 = efs1.fit(X_train, y_train) Features: 15/15 print('Selected features:', efs1.best_idx_) Selected features: (2, 3) # Generate the new subsets based on the selected features # Note that the transform call is equivalent to # X_train[:, efs1.k_feature_idx_] X_train_efs = efs1.transform(X_train) X_test_efs = efs1.transform(X_test) # Fit the estimator using the new feature subset # and make a prediction on the test data knn.fit(X_train_efs, y_train) y_pred = knn.predict(X_test_efs) # Compute the accuracy of the prediction acc = float((y_test == y_pred).sum()) / y_pred.shape[0] print('Test set accuracy: %.2f %%' % (acc*100)) Test set accuracy: 96.00 %","title":"Example 5 - Using the selected feature subset For making new predictions"},{"location":"user_guide/feature_selection/ExhaustiveFeatureSelector/#example-6-exhaustive-feature-selection-and-gridsearch","text":"# Initialize the dataset from sklearn.datasets import load_iris from sklearn.model_selection import train_test_split iris = load_iris() X, y = iris.data, iris.target X_train, X_test, y_train, y_test = train_test_split( X, y, test_size=0.33, random_state=1) Use scikit-learn's GridSearch to tune the hyperparameters of the LogisticRegression estimator inside the ExhaustiveFeatureSelector and use it for prediction in the pipeline. Note that the clone_estimator attribute needs to be set to False . from sklearn.model_selection import GridSearchCV from sklearn.pipeline import make_pipeline from sklearn.linear_model import LogisticRegression from mlxtend.feature_selection import ExhaustiveFeatureSelector as EFS lr = LogisticRegression(multi_class='multinomial', solver='newton-cg', random_state=123) efs1 = EFS(estimator=lr, min_features=2, max_features=3, scoring='accuracy', print_progress=False, clone_estimator=False, cv=5, n_jobs=1) pipe = make_pipeline(efs1, lr) param_grid = {'exhaustivefeatureselector__estimator__C': [0.1, 1.0, 10.0]} gs = GridSearchCV(estimator=pipe, param_grid=param_grid, scoring='accuracy', n_jobs=1, cv=2, verbose=1, refit=False) # run gridearch gs = gs.fit(X_train, y_train) Fitting 2 folds for each of 3 candidates, totalling 6 fits ... and the \"best\" parameters determined by GridSearch are ... print(\"Best parameters via GridSearch\", gs.best_params_) Best parameters via GridSearch {'exhaustivefeatureselector__estimator__C': 0.1}","title":"Example 6 - Exhaustive feature selection and GridSearch"},{"location":"user_guide/feature_selection/ExhaustiveFeatureSelector/#obtaining-the-best-k-feature-indices-after-gridsearch","text":"If we are interested in the best k best feature indices via SequentialFeatureSelection.best_idx_ , we have to initialize a GridSearchCV object with refit=True . Now, the grid search object will take the complete training dataset and the best parameters, which it found via cross-validation, to train the estimator pipeline. gs = GridSearchCV(estimator=pipe, param_grid=param_grid, scoring='accuracy', n_jobs=1, cv=2, verbose=1, refit=True) After running the grid search, we can access the individual pipeline objects of the best_estimator_ via the steps attribute. gs = gs.fit(X_train, y_train) gs.best_estimator_.steps Fitting 2 folds for each of 3 candidates, totalling 6 fits [('exhaustivefeatureselector', ExhaustiveFeatureSelector(clone_estimator=False, estimator=LogisticRegression(C=0.1, multi_class='multinomial', random_state=123, solver='newton-cg'), feature_groups=[[0], [1], [2], [3]], max_features=3, min_features=2, print_progress=False)), ('logisticregression', LogisticRegression(multi_class='multinomial', random_state=123, solver='newton-cg'))] Via sub-indexing, we can then obtain the best-selected feature subset: print('Best features:', gs.best_estimator_.steps[0][1].best_idx_) Best features: (2, 3) During cross-validation, this feature combination had a CV accuracy of: print('Best score:', gs.best_score_) Best score: 0.96 gs.best_params_ {'exhaustivefeatureselector__estimator__C': 0.1} Alternatively , if we can set the \"best grid search parameters\" in our pipeline manually if we ran GridSearchCV with refit=False . It should yield the same results: pipe.set_params(**gs.best_params_).fit(X_train, y_train) print('Best features:', pipe.steps[0][1].best_idx_) Best features: (2, 3)","title":"Obtaining the best k feature indices after GridSearch"},{"location":"user_guide/feature_selection/ExhaustiveFeatureSelector/#example-7-exhaustive-feature-selection-with-loocv","text":"The ExhaustiveFeatureSelector is not restricted to k-fold cross-validation. You can use any type of cross-validation method that supports the general scikit-learn cross-validation API. The following example illustrates the use of scikit-learn's LeaveOneOut cross-validation method in combination with the exhaustive feature selector. from sklearn.neighbors import KNeighborsClassifier from sklearn.datasets import load_iris from mlxtend.feature_selection import ExhaustiveFeatureSelector as EFS from sklearn.model_selection import LeaveOneOut iris = load_iris() X = iris.data y = iris.target knn = KNeighborsClassifier(n_neighbors=3) efs1 = EFS(knn, min_features=1, max_features=4, scoring='accuracy', print_progress=True, cv=LeaveOneOut()) ### Use cross-validation generator here efs1 = efs1.fit(X, y) print('Best accuracy score: %.2f' % efs1.best_score_) print('Best subset (indices):', efs1.best_idx_) print('Best subset (corresponding names):', efs1.best_feature_names_) Features: 15/15 Best accuracy score: 0.96 Best subset (indices): (3,) Best subset (corresponding names): ('3',)","title":"Example 7 - Exhaustive Feature Selection with LOOCV"},{"location":"user_guide/feature_selection/ExhaustiveFeatureSelector/#example-8-interrupting-long-runs-for-intermediate-results","text":"If your run is taking too long, it is possible to trigger a KeyboardInterrupt (e.g., ctrl+c on a Mac, or interrupting the cell in a Jupyter notebook) to obtain temporary results. Toy dataset from sklearn.datasets import make_classification from sklearn.model_selection import train_test_split X, y = make_classification( n_samples=200000, n_features=6, n_informative=2, n_redundant=1, n_repeated=1, n_clusters_per_class=2, flip_y=0.05, class_sep=0.5, random_state=123, ) X_train, X_test, y_train, y_test = train_test_split( X, y, test_size=0.2, random_state=123 ) Long run with interruption from mlxtend.feature_selection import ExhaustiveFeatureSelector as EFS from sklearn.linear_model import LogisticRegression model = LogisticRegression(max_iter=10000) efs1 = EFS(model, min_features=1, max_features=4, print_progress=True, scoring='accuracy') efs1 = efs1.fit(X_train, y_train) Features: 56/56 Finalizing the fit Note that the feature selection run hasn't finished, so certain attributes may not be available. In order to use the EFS instance, it is recommended to call finalize_fit , which will make EFS estimator appear as \"fitted\" process the temporary results: efs1.finalize_fit() print('Best accuracy score: %.2f' % efs1.best_score_) print('Best subset (indices):', efs1.best_idx_) Best accuracy score: 0.73 Best subset (indices): (1, 2)","title":"Example 8 - Interrupting Long Runs for Intermediate Results"},{"location":"user_guide/feature_selection/ExhaustiveFeatureSelector/#example-9-working-with-feature-groups","text":"Since mlxtend v0.21.0, it is possible to specify feature groups. Feature groups allow you to group certain features together, such that they are always selected as a group. This can be very useful in contexts similar to one-hot encoding -- if you want to treat the one-hot encoded feature as a single feature: In the following example, we specify sepal length and sepal width as a feature group so that they are always selected together: from sklearn.datasets import load_iris import pandas as pd iris = load_iris() X = iris.data y = iris.target X_df = pd.DataFrame(X, columns=['sepal len', 'petal len', 'sepal wid', 'petal wid']) X_df.head() .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } sepal len petal len sepal wid petal wid 0 5.1 3.5 1.4 0.2 1 4.9 3.0 1.4 0.2 2 4.7 3.2 1.3 0.2 3 4.6 3.1 1.5 0.2 4 5.0 3.6 1.4 0.2 from sklearn.neighbors import KNeighborsClassifier from mlxtend.feature_selection import ExhaustiveFeatureSelector as EFS knn = KNeighborsClassifier(n_neighbors=3) efs1 = EFS(knn, min_features=2, max_features=2, scoring='accuracy', feature_groups=[['sepal len', 'sepal wid'], ['petal len'], ['petal wid']], cv=3) efs1 = efs1.fit(X_df, y) print('Best accuracy score: %.2f' % efs1.best_score_) print('Best subset (indices):', efs1.best_idx_) print('Best subset (corresponding names):', efs1.best_feature_names_) Features: 3/3 Best accuracy score: 0.97 Best subset (indices): (0, 2, 3) Best subset (corresponding names): ('sepal len', 'sepal wid', 'petal wid') Notice that the returned number of features is 3, since the number of min_features and max_features corresponds to the number of feature groups. I.e., we have 2 feature groups in ['sepal len', 'sepal wid'], ['petal wid'] , but it expands to 3 features.","title":"Example 9 - Working with Feature Groups"},{"location":"user_guide/feature_selection/ExhaustiveFeatureSelector/#api","text":"ExhaustiveFeatureSelector(estimator, min_features=1, max_features=1, print_progress=True, scoring='accuracy', cv=5, n_jobs=1, pre_dispatch='2 n_jobs', clone_estimator=True, fixed_features=None, feature_groups=None)* Exhaustive Feature Selection for Classification and Regression. (new in v0.4.3) Parameters estimator : scikit-learn classifier or regressor min_features : int (default: 1) Minumum number of features to select max_features : int (default: 1) Maximum number of features to select. If parameter feature_groups is not None, the number of features is equal to the number of feature groups, i.e. len(feature_groups) . For example, if feature_groups = [[0], [1], [2, 3], [4]] , then the max_features value cannot exceed 4. print_progress : bool (default: True) Prints progress as the number of epochs to stderr. scoring : str, (default='accuracy') Scoring metric in {accuracy, f1, precision, recall, roc_auc} for classifiers, {'mean_absolute_error', 'mean_squared_error', 'median_absolute_error', 'r2'} for regressors, or a callable object or function with signature scorer(estimator, X, y) . cv : int (default: 5) Scikit-learn cross-validation generator or int . If estimator is a classifier (or y consists of integer class labels), stratified k-fold is performed, and regular k-fold cross-validation otherwise. No cross-validation if cv is None, False, or 0. n_jobs : int (default: 1) The number of CPUs to use for evaluating different feature subsets in parallel. -1 means 'all CPUs'. pre_dispatch : int, or string (default: '2*n_jobs') Controls the number of jobs that get dispatched during parallel execution if n_jobs > 1 or n_jobs=-1 . Reducing this number can be useful to avoid an explosion of memory consumption when more jobs get dispatched than CPUs can process. This parameter can be: None, in which case all the jobs are immediately created and spawned. Use this for lightweight and fast-running jobs, to avoid delays due to on-demand spawning of the jobs An int, giving the exact number of total jobs that are spawned A string, giving an expression as a function of n_jobs, as in 2*n_jobs clone_estimator : bool (default: True) Clones estimator if True; works with the original estimator instance if False. Set to False if the estimator doesn't implement scikit-learn's set_params and get_params methods. In addition, it is required to set cv=0, and n_jobs=1. fixed_features : tuple (default: None) If not None , the feature indices provided as a tuple will be regarded as fixed by the feature selector. For example, if fixed_features=(1, 3, 7) , the 2nd, 4th, and 8th feature are guaranteed to be present in the solution. Note that if fixed_features is not None , make sure that the number of features to be selected is greater than len(fixed_features) . In other words, ensure that k_features > len(fixed_features) . feature_groups : list or None (default: None) Optional argument for treating certain features as a group. This means, the features within a group are always selected together, never split. For example, feature_groups=[[1], [2], [3, 4, 5]] specifies 3 feature groups.In this case, possible feature selection results with k_features=2 are [[1], [2] , [[1], [3, 4, 5]] , or [[2], [3, 4, 5]] . Feature groups can be useful for interpretability, for example, if features 3, 4, 5 are one-hot encoded features. (For more details, please read the notes at the bottom of this docstring). New in mlxtend v. 0.21.0. Attributes best_idx_ : array-like, shape = [n_predictions] Feature Indices of the selected feature subsets. best_feature_names_ : array-like, shape = [n_predictions] Feature names of the selected feature subsets. If pandas DataFrames are used in the fit method, the feature names correspond to the column names. Otherwise, the feature names are string representation of the feature array indices. New in v 0.13.0. best_score_ : float Cross validation average score of the selected subset. subsets_ : dict A dictionary of selected feature subsets during the exhaustive selection, where the dictionary keys are the lengths k of these feature subsets. The dictionary values are dictionaries themselves with the following keys: 'feature_idx' (tuple of indices of the feature subset) 'feature_names' (tuple of feature names of the feat. subset) 'cv_scores' (list individual cross-validation scores) 'avg_score' (average cross-validation score) Note that if pandas DataFrames are used in the fit method, the 'feature_names' correspond to the column names. Otherwise, the feature names are string representation of the feature array indices. The 'feature_names' is new in v. 0.13.0. Notes (1) If parameter feature_groups is not None, the number of features is equal to the number of feature groups, i.e. len(feature_groups) . For example, if feature_groups = [[0], [1], [2, 3], [4]] , then the max_features value cannot exceed 4. (2) Although two or more individual features may be considered as one group throughout the feature-selection process, it does not mean the individual features of that group have the same impact on the outcome. For instance, in linear regression, the coefficient of the feature 2 and 3 can be different even if they are considered as one group in feature_groups. (3) If both fixed_features and feature_groups are specified, ensure that each feature group contains the fixed_features selection. E.g., for a 3-feature set fixed_features=[0, 1] and feature_groups=[[0, 1], [2]] is valid; fixed_features=[0, 1] and feature_groups=[[0], [1, 2]] is not valid. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/feature_selection/ExhaustiveFeatureSelector/","title":"API"},{"location":"user_guide/feature_selection/ExhaustiveFeatureSelector/#methods","text":"fit(X, y, groups=None, fit_params) Perform feature selection and learn model from training data. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. New in v 0.13.0: pandas DataFrames are now also accepted as argument for X. y : array-like, shape = [n_samples] Target values. groups : array-like, with shape (n_samples,), optional Group labels for the samples used while splitting the dataset into train/test set. Passed to the fit method of the cross-validator. fit_params : dict of string -> object, optional Parameters to pass to to the fit method of classifier. Returns self : object fit_transform(X, y, groups=None, fit_params) Fit to training data and return the best selected features from X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. New in v 0.13.0: pandas DataFrames are now also accepted as argument for X. y : array-like, shape = [n_samples] Target values. groups : array-like, with shape (n_samples,), optional Group labels for the samples used while splitting the dataset into train/test set. Passed to the fit method of the cross-validator. fit_params : dict of string -> object, optional Parameters to pass to to the fit method of classifier. Returns Feature subset of X, shape={n_samples, k_features} get_metric_dict(confidence_interval=0.95) Return metric dictionary Parameters confidence_interval : float (default: 0.95) A positive float between 0.0 and 1.0 to compute the confidence interval bounds of the CV score averages. Returns Dictionary with items where each dictionary value is a list with the number of iterations (number of feature subsets) as its length. The dictionary keys corresponding to these lists are as follows: 'feature_idx': tuple of the indices of the feature subset 'cv_scores': list with individual CV scores 'avg_score': of CV average scores 'std_dev': standard deviation of the CV score average 'std_err': standard error of the CV score average 'ci_bound': confidence interval bound of the CV score average get_params(deep=True) Get parameters for this estimator. Parameters deep : bool, default=True If True, will return the parameters for this estimator and contained subobjects that are estimators. Returns params : dict Parameter names mapped to their values. set_params( params) Set the parameters of this estimator. The method works on simple estimators as well as on nested objects (such as :class:`~sklearn.pipeline.Pipeline`). The latter have parameters of the form ``<component>__<parameter>`` so that it's possible to update each component of a nested object. Parameters **params : dict Estimator parameters. Returns self : estimator instance Estimator instance. transform(X) Return the best selected features from X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. New in v 0.13.0: pandas DataFrames are now also accepted as argument for X. Returns Feature subset of X, shape={n_samples, k_features}","title":"Methods"},{"location":"user_guide/feature_selection/SequentialFeatureSelector/","text":"SequentialFeatureSelector: The popular forward and backward feature selection approaches (including floating variants) Implementation of sequential feature algorithms (SFAs) -- greedy search algorithms -- that have been developed as a suboptimal solution to the computationally often not feasible exhaustive search. from mlxtend.feature_selection import SequentialFeatureSelector Overview Sequential feature selection algorithms are a family of greedy search algorithms that are used to reduce an initial d -dimensional feature space to a k -dimensional feature subspace where k < d . The motivation behind feature selection algorithms is to automatically select a subset of features most relevant to the problem. The goal of feature selection is two-fold: We want to improve the computational efficiency and reduce the model's generalization error by removing irrelevant features or noise. In addition, a wrapper approach such as sequential feature selection is advantageous if embedded feature selection -- for example, a regularization penalty like LASSO -- is not applicable. In a nutshell, SFAs remove or add one feature at a time based on the classifier performance until a feature subset of the desired size k is reached. There are four different flavors of SFAs available via the SequentialFeatureSelector : Sequential Forward Selection (SFS) Sequential Backward Selection (SBS) Sequential Forward Floating Selection (SFFS) Sequential Backward Floating Selection (SBFS) The floating variants, SFFS and SBFS, can be considered extensions to the simpler SFS and SBS algorithms. The floating algorithms have an additional exclusion or inclusion step to remove features once they were included (or excluded) so that a larger number of feature subset combinations can be sampled. It is important to emphasize that this step is conditional and only occurs if the resulting feature subset is assessed as \"better\" by the criterion function after the removal (or addition) of a particular feature. Furthermore, I added an optional check to skip the conditional exclusion steps if the algorithm gets stuck in cycles. How is this different from Recursive Feature Elimination (RFE) -- e.g., as implemented in sklearn.feature_selection.RFE ? RFE is computationally less complex using the feature weight coefficients (e.g., linear models) or feature importance (tree-based algorithms) to eliminate features recursively, whereas SFSs eliminate (or add) features based on a user-defined classifier/regression performance metric. Tutorial Videos Visual Illustration A visual illustration of the sequential backward selection process is provided below, from the paper Joe Bemister-Buffington, Alex J. Wolf, Sebastian Raschka, and Leslie A. Kuhn (2020) Machine Learning to Identify Flexibility Signatures of Class A GPCR Inhibition Biomolecules 2020, 10, 454. https://www.mdpi.com/2218-273X/10/3/454# Algorithmic Details Sequential Forward Selection (SFS) Input: Y = \\{y_1, y_2, ..., y_d\\} The SFS algorithm takes the whole d -dimensional feature set as input. Output: X_k = \\{x_j \\; | \\;j = 1, 2, ..., k; \\; x_j \\in Y\\} , where k = (0, 1, 2, ..., d) SFS returns a subset of features; the number of selected features k , where k < d , has to be specified a priori . Initialization: X_0 = \\emptyset , k = 0 We initialize the algorithm with an empty set \\emptyset (\"null set\") so that k = 0 (where k is the size of the subset). Step 1 (Inclusion): x^+ = \\text{ arg max } J(X_k + x), \\text{ where } x \\in Y - X_k X_{k+1} = X_k + x^+ k = k + 1 Go to Step 1 in this step, we add an additional feature, x^+ , to our feature subset X_k . x^+ is the feature that maximizes our criterion function, that is, the feature that is associated with the best classifier performance if it is added to X_k . We repeat this procedure until the termination criterion is satisfied. Termination: k = p We add features from the feature subset X_k until the feature subset of size k contains the number of desired features p that we specified a priori . Sequential Backward Selection (SBS) Input: the set of all features, Y = \\{y_1, y_2, ..., y_d\\} The SBS algorithm takes the whole feature set as input. Output: X_k = \\{x_j \\; | \\;j = 1, 2, ..., k; \\; x_j \\in Y\\} , where k = (0, 1, 2, ..., d) SBS returns a subset of features; the number of selected features k , where k < d , has to be specified a priori . Initialization: X_0 = Y , k = d We initialize the algorithm with the given feature set so that the k = d . Step 1 (Exclusion): x^- = \\text{ arg max } J(X_k - x), \\text{ where } x \\in X_k X_{k-1} = X_k - x^- k = k - 1 Go to Step 1 In this step, we remove a feature, x^- from our feature subset X_k . x^- is the feature that maximizes our criterion function upon re,oval, that is, the feature that is associated with the best classifier performance if it is removed from X_k . We repeat this procedure until the termination criterion is satisfied. Termination: k = p We add features from the feature subset X_k until the feature subset of size k contains the number of desired features p that we specified a priori . Sequential Backward Floating Selection (SBFS) Input: the set of all features, Y = \\{y_1, y_2, ..., y_d\\} The SBFS algorithm takes the whole feature set as input. Output: X_k = \\{x_j \\; | \\;j = 1, 2, ..., k; \\; x_j \\in Y\\} , where k = (0, 1, 2, ..., d) SBFS returns a subset of features; the number of selected features k , where k < d , has to be specified a priori . Initialization: X_0 = Y , k = d We initialize the algorithm with the given feature set so that the k = d . Step 1 (Exclusion): x^- = \\text{ arg max } J(X_k - x), \\text{ where } x \\in X_k X_{k-1} = X_k - x^- k = k - 1 Go to Step 2 In this step, we remove a feature, x^- from our feature subset X_k . x^- is the feature that maximizes our criterion function upon removal, that is, the feature that is associated with the best classifier performance if it is removed from X_k . Step 2 (Conditional Inclusion): x^+ = \\text{ arg max } J(X_k + x), \\text{ where } x \\in Y - X_k if J(X_k + x) > J(X_k) : X_{k+1} = X_k + x^+ k = k + 1 Go to Step 1 In Step 2, we search for features that improve the classifier performance if they are added back to the feature subset. If such features exist, we add the feature x^+ for which the performance improvement is maximized. If k = 2 or an improvement cannot be made (i.e., such feature x^+ cannot be found), go back to step 1; else, repeat this step. Termination: k = p We add features from the feature subset X_k until the feature subset of size k contains the number of desired features p that we specified a priori . Sequential Forward Floating Selection (SFFS) Input: the set of all features, Y = \\{y_1, y_2, ..., y_d\\} The SFFS algorithm takes the whole feature set as input, if our feature space consists of, e.g. 10, if our feature space consists of 10 dimensions ( d = 10 ). Output: a subset of features, X_k = \\{x_j \\; | \\;j = 1, 2, ..., k; \\; x_j \\in Y\\} , where k = (0, 1, 2, ..., d) The returned output of the algorithm is a subset of the feature space of a specified size. E.g., a subset of 5 features from a 10-dimensional feature space ( k = 5, d = 10 ). Initialization: X_0 = \\emptyset , k = 0 We initialize the algorithm with an empty set (\"null set\") so that the k = 0 (where k is the size of the subset) Step 1 (Inclusion): x^+ = \\text{ arg max } J(X_k + x), \\text{ where } x \\in Y - X_k X_{k+1} = X_k + x^+ k = k + 1 Go to Step 2 Step 2 (Conditional Exclusion): x^- = \\text{ arg max } J(X_k - x), \\text{ where } x \\in X_k if \\; J(X_k - x) > J(X_k) : X_{k-1} = X_k - x^- k = k - 1 Go to Step 1 In step 1, we include the feature from the feature space that leads to the best performance increase for our feature subset (assessed by the criterion function ). Then, we go over to step 2 In step 2, we only remove a feature if the resulting subset would gain an increase in performance. If k = 2 or an improvement cannot be made (i.e., such feature x^+ cannot be found), go back to step 1; else, repeat this step. Steps 1 and 2 are repeated until the Termination criterion is reached. Termination: stop when k equals the number of desired features References Ferri, F. J., Pudil P., Hatef, M., Kittler, J. (1994). \"Comparative study of techniques for large-scale feature selection.\" Pattern Recognition in Practice IV : 403-413. Pudil, P., Novovi\u010dov\u00e1, J., & Kittler, J. (1994). \"Floating search methods in feature selection.\" Pattern recognition letters 15.11 (1994): 1119-1125. Example 1 - A simple Sequential Forward Selection example Initializing a simple classifier from scikit-learn: from sklearn.neighbors import KNeighborsClassifier from sklearn.datasets import load_iris iris = load_iris() X = iris.data y = iris.target knn = KNeighborsClassifier(n_neighbors=4) We start by selection the \"best\" 3 features from the Iris dataset via Sequential Forward Selection (SFS). Here, we set forward=True and floating=False . By choosing cv=0 , we don't perform any cross-validation, therefore, the performance (here: 'accuracy' ) is computed entirely on the training set. from mlxtend.feature_selection import SequentialFeatureSelector as SFS sfs1 = SFS(knn, k_features=3, forward=True, floating=False, verbose=2, scoring='accuracy', cv=0) sfs1 = sfs1.fit(X, y) [Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers. [Parallel(n_jobs=1)]: Done 1 out of 1 | elapsed: 0.0s remaining: 0.0s [Parallel(n_jobs=1)]: Done 4 out of 4 | elapsed: 0.0s finished [2023-05-17 08:36:17] Features: 1/3 -- score: 0.96[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers. [Parallel(n_jobs=1)]: Done 1 out of 1 | elapsed: 0.0s remaining: 0.0s [Parallel(n_jobs=1)]: Done 3 out of 3 | elapsed: 0.0s finished [2023-05-17 08:36:17] Features: 2/3 -- score: 0.9733333333333334[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers. [Parallel(n_jobs=1)]: Done 1 out of 1 | elapsed: 0.0s remaining: 0.0s [Parallel(n_jobs=1)]: Done 2 out of 2 | elapsed: 0.0s finished [2023-05-17 08:36:17] Features: 3/3 -- score: 0.9733333333333334 Via the subsets_ attribute, we can take a look at the selected feature indices at each step: sfs1.subsets_ {1: {'feature_idx': (3,), 'cv_scores': array([0.96]), 'avg_score': 0.96, 'feature_names': ('3',)}, 2: {'feature_idx': (2, 3), 'cv_scores': array([0.97333333]), 'avg_score': 0.9733333333333334, 'feature_names': ('2', '3')}, 3: {'feature_idx': (1, 2, 3), 'cv_scores': array([0.97333333]), 'avg_score': 0.9733333333333334, 'feature_names': ('1', '2', '3')}} sfs1 = sfs1.fit(X, y) sfs1.subsets_ [Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers. [Parallel(n_jobs=1)]: Done 1 out of 1 | elapsed: 0.0s remaining: 0.0s [Parallel(n_jobs=1)]: Done 4 out of 4 | elapsed: 0.0s finished [2023-05-17 08:36:17] Features: 1/3 -- score: 0.96[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers. [Parallel(n_jobs=1)]: Done 1 out of 1 | elapsed: 0.0s remaining: 0.0s [Parallel(n_jobs=1)]: Done 3 out of 3 | elapsed: 0.0s finished [2023-05-17 08:36:17] Features: 2/3 -- score: 0.9733333333333334[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers. [Parallel(n_jobs=1)]: Done 1 out of 1 | elapsed: 0.0s remaining: 0.0s [Parallel(n_jobs=1)]: Done 2 out of 2 | elapsed: 0.0s finished [2023-05-17 08:36:17] Features: 3/3 -- score: 0.9733333333333334 {1: {'feature_idx': (3,), 'cv_scores': array([0.96]), 'avg_score': 0.96, 'feature_names': ('3',)}, 2: {'feature_idx': (2, 3), 'cv_scores': array([0.97333333]), 'avg_score': 0.9733333333333334, 'feature_names': ('2', '3')}, 3: {'feature_idx': (1, 2, 3), 'cv_scores': array([0.97333333]), 'avg_score': 0.9733333333333334, 'feature_names': ('1', '2', '3')}} Furthermore, we can access the indices of the 3 best features directly via the k_feature_idx_ attribute: sfs1.k_feature_idx_ (1, 2, 3) Finally, the prediction score for these 3 features can be accesses via k_score_ : sfs1.k_score_ 0.9733333333333334 Feature Names When working with large datasets, the feature indices might be hard to interpret. In this case, we recommend using pandas DataFrames with distinct column names as input: import pandas as pd df_X = pd.DataFrame(X, columns=[\"Sepal length\", \"Sepal width\", \"Petal length\", \"Petal width\"]) df_X.head() .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } Sepal length Sepal width Petal length Petal width 0 5.1 3.5 1.4 0.2 1 4.9 3.0 1.4 0.2 2 4.7 3.2 1.3 0.2 3 4.6 3.1 1.5 0.2 4 5.0 3.6 1.4 0.2 sfs1 = sfs1.fit(df_X, y) print('Best accuracy score: %.2f' % sfs1.k_score_) print('Best subset (indices):', sfs1.k_feature_idx_) print('Best subset (corresponding names):', sfs1.k_feature_names_) [Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers. Best accuracy score: 0.97 Best subset (indices): (1, 2, 3) Best subset (corresponding names): ('Sepal width', 'Petal length', 'Petal width') [Parallel(n_jobs=1)]: Done 1 out of 1 | elapsed: 0.0s remaining: 0.0s [Parallel(n_jobs=1)]: Done 4 out of 4 | elapsed: 0.0s finished [2023-05-17 08:36:17] Features: 1/3 -- score: 0.96[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers. [Parallel(n_jobs=1)]: Done 1 out of 1 | elapsed: 0.0s remaining: 0.0s [Parallel(n_jobs=1)]: Done 3 out of 3 | elapsed: 0.0s finished [2023-05-17 08:36:17] Features: 2/3 -- score: 0.9733333333333334[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers. [Parallel(n_jobs=1)]: Done 1 out of 1 | elapsed: 0.0s remaining: 0.0s [Parallel(n_jobs=1)]: Done 2 out of 2 | elapsed: 0.0s finished [2023-05-17 08:36:17] Features: 3/3 -- score: 0.9733333333333334 Example 2 - Toggling between SFS, SBS, SFFS, and SBFS Using the forward and floating parameters, we can toggle between SFS, SBS, SFFS, and SBFS as shown below. Note that we are performing (stratified) 4-fold cross-validation for more robust estimates in contrast to Example 1. Via n_jobs=-1 , we choose to run the cross-validation on all our available CPU cores. # Sequential Forward Selection sfs = SFS(knn, k_features=3, forward=True, floating=False, scoring='accuracy', cv=4, n_jobs=-1) sfs = sfs.fit(X, y) print('\\nSequential Forward Selection (k=3):') print(sfs.k_feature_idx_) print('CV Score:') print(sfs.k_score_) ################################################### # Sequential Backward Selection sbs = SFS(knn, k_features=3, forward=False, floating=False, scoring='accuracy', cv=4, n_jobs=-1) sbs = sbs.fit(X, y) print('\\nSequential Backward Selection (k=3):') print(sbs.k_feature_idx_) print('CV Score:') print(sbs.k_score_) ################################################### # Sequential Forward Floating Selection sffs = SFS(knn, k_features=3, forward=True, floating=True, scoring='accuracy', cv=4, n_jobs=-1) sffs = sffs.fit(X, y) print('\\nSequential Forward Floating Selection (k=3):') print(sffs.k_feature_idx_) print('CV Score:') print(sffs.k_score_) ################################################### # Sequential Backward Floating Selection sbfs = SFS(knn, k_features=3, forward=False, floating=True, scoring='accuracy', cv=4, n_jobs=-1) sbfs = sbfs.fit(X, y) print('\\nSequential Backward Floating Selection (k=3):') print(sbfs.k_feature_idx_) print('CV Score:') print(sbfs.k_score_) Sequential Forward Selection (k=3): (1, 2, 3) CV Score: 0.9731507823613088 Sequential Backward Selection (k=3): (1, 2, 3) CV Score: 0.9731507823613088 Sequential Forward Floating Selection (k=3): (1, 2, 3) CV Score: 0.9731507823613088 Sequential Backward Floating Selection (k=3): (1, 2, 3) CV Score: 0.9731507823613088 In this simple scenario, selecting the best 3 features out of the 4 available features in the Iris set, we end up with similar results regardless of which sequential selection algorithms we used. Example 3 - Visualizing the results in DataFrames For our convenience, we can visualize the output from the feature selection in a pandas DataFrame format using the get_metric_dict method of the SequentialFeatureSelector object. The columns std_dev and std_err represent the standard deviation and standard errors of the cross-validation scores, respectively. Below, we see the DataFrame of the Sequential Forward Selector from Example 2: import pandas as pd pd.DataFrame.from_dict(sfs.get_metric_dict()).T .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } feature_idx cv_scores avg_score feature_names ci_bound std_dev std_err 1 (3,) [0.9736842105263158, 0.9473684210526315, 0.918... 0.959993 (3,) 0.048319 0.030143 0.017403 2 (2, 3) [0.9736842105263158, 0.9473684210526315, 0.918... 0.959993 (2, 3) 0.048319 0.030143 0.017403 3 (1, 2, 3) [0.9736842105263158, 1.0, 0.9459459459459459, ... 0.973151 (1, 2, 3) 0.030639 0.019113 0.011035 Now, let's compare it to the Sequential Backward Selector: pd.DataFrame.from_dict(sbs.get_metric_dict()).T .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } feature_idx cv_scores avg_score feature_names ci_bound std_dev std_err 4 (0, 1, 2, 3) [0.9736842105263158, 0.9473684210526315, 0.918... 0.953236 (0, 1, 2, 3) 0.03602 0.022471 0.012974 3 (1, 2, 3) [0.9736842105263158, 1.0, 0.9459459459459459, ... 0.973151 (1, 2, 3) 0.030639 0.019113 0.011035 We can see that both SFS and SBFS found the same \"best\" 3 features, however, the intermediate steps where obviously different. The ci_bound column in the DataFrames above represents the confidence interval around the computed cross-validation scores. By default, a confidence interval of 95% is used, but we can use different confidence bounds via the confidence_interval parameter. E.g., the confidence bounds for a 90% confidence interval can be obtained as follows: pd.DataFrame.from_dict(sbs.get_metric_dict(confidence_interval=0.90)).T .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } feature_idx cv_scores avg_score feature_names ci_bound std_dev std_err 4 (0, 1, 2, 3) [0.9736842105263158, 0.9473684210526315, 0.918... 0.953236 (0, 1, 2, 3) 0.027658 0.022471 0.012974 3 (1, 2, 3) [0.9736842105263158, 1.0, 0.9459459459459459, ... 0.973151 (1, 2, 3) 0.023525 0.019113 0.011035 Example 4 - Plotting the results After importing the little helper function plotting.plot_sequential_feature_selection , we can also visualize the results using matplotlib figures. from mlxtend.plotting import plot_sequential_feature_selection as plot_sfs import matplotlib.pyplot as plt sfs = SFS(knn, k_features=4, forward=True, floating=False, scoring='accuracy', verbose=2, cv=5) sfs = sfs.fit(X, y) fig1 = plot_sfs(sfs.get_metric_dict(), kind='std_dev') plt.ylim([0.8, 1]) plt.title('Sequential Forward Selection (w. StdDev)') plt.grid() plt.show() [Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers. [Parallel(n_jobs=1)]: Done 1 out of 1 | elapsed: 0.0s remaining: 0.0s [Parallel(n_jobs=1)]: Done 4 out of 4 | elapsed: 0.0s finished [2023-05-17 08:36:18] Features: 1/4 -- score: 0.96[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers. [Parallel(n_jobs=1)]: Done 1 out of 1 | elapsed: 0.0s remaining: 0.0s [Parallel(n_jobs=1)]: Done 3 out of 3 | elapsed: 0.0s finished [2023-05-17 08:36:18] Features: 2/4 -- score: 0.9666666666666668[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers. [Parallel(n_jobs=1)]: Done 1 out of 1 | elapsed: 0.0s remaining: 0.0s [Parallel(n_jobs=1)]: Done 2 out of 2 | elapsed: 0.0s finished [2023-05-17 08:36:18] Features: 3/4 -- score: 0.9533333333333334[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers. [Parallel(n_jobs=1)]: Done 1 out of 1 | elapsed: 0.0s remaining: 0.0s [Parallel(n_jobs=1)]: Done 1 out of 1 | elapsed: 0.0s finished [2023-05-17 08:36:18] Features: 4/4 -- score: 0.9733333333333334 Example 5 - Sequential Feature Selection for Regression Similar to the classification examples above, the SequentialFeatureSelector also supports scikit-learn's estimators for regression. from sklearn.linear_model import LinearRegression from sklearn.datasets import fetch_california_housing data = fetch_california_housing() X, y = data.data, data.target lr = LinearRegression() sfs = SFS(lr, k_features=8, forward=True, floating=False, scoring='neg_mean_squared_error', cv=10) sfs = sfs.fit(X, y) fig = plot_sfs(sfs.get_metric_dict(), kind='std_err') plt.title('Sequential Forward Selection (w. StdErr)') plt.grid() plt.show() Example 6 -- Feature Selection with Fixed Train/Validation Splits If you do not wish to use cross-validation (here: k-fold cross-validation, i.e., rotating training and validation folds), you can use the PredefinedHoldoutSplit class to specify your own, fixed training and validation split. from sklearn.datasets import load_iris from mlxtend.evaluate import PredefinedHoldoutSplit import numpy as np iris = load_iris() X = iris.data y = iris.target rng = np.random.RandomState(123) my_validation_indices = rng.permutation(np.arange(150))[:30] print(my_validation_indices) [ 72 112 132 88 37 138 87 42 8 90 141 33 59 116 135 104 36 13 63 45 28 133 24 127 46 20 31 121 117 4] from sklearn.neighbors import KNeighborsClassifier from mlxtend.feature_selection import SequentialFeatureSelector as SFS knn = KNeighborsClassifier(n_neighbors=4) piter = PredefinedHoldoutSplit(my_validation_indices) sfs1 = SFS(knn, k_features=3, forward=True, floating=False, verbose=2, scoring='accuracy', cv=piter) sfs1 = sfs1.fit(X, y) [Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers. [Parallel(n_jobs=1)]: Done 1 out of 1 | elapsed: 0.0s remaining: 0.0s [Parallel(n_jobs=1)]: Done 4 out of 4 | elapsed: 0.0s finished [2023-05-17 08:36:19] Features: 1/3 -- score: 0.9666666666666667[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers. [Parallel(n_jobs=1)]: Done 1 out of 1 | elapsed: 0.0s remaining: 0.0s [Parallel(n_jobs=1)]: Done 3 out of 3 | elapsed: 0.0s finished [2023-05-17 08:36:19] Features: 2/3 -- score: 0.9666666666666667[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers. [Parallel(n_jobs=1)]: Done 1 out of 1 | elapsed: 0.0s remaining: 0.0s [Parallel(n_jobs=1)]: Done 2 out of 2 | elapsed: 0.0s finished [2023-05-17 08:36:19] Features: 3/3 -- score: 0.9666666666666667 Example 7 -- Using the Selected Feature Subset For Making New Predictions # Initialize the dataset from sklearn.neighbors import KNeighborsClassifier from sklearn.datasets import load_iris from sklearn.model_selection import train_test_split iris = load_iris() X, y = iris.data, iris.target X_train, X_test, y_train, y_test = train_test_split( X, y, test_size=0.33, random_state=1) knn = KNeighborsClassifier(n_neighbors=4) # Select the \"best\" three features via # 5-fold cross-validation on the training set. from mlxtend.feature_selection import SequentialFeatureSelector as SFS sfs1 = SFS(knn, k_features=3, forward=True, floating=False, scoring='accuracy', cv=5) sfs1 = sfs1.fit(X_train, y_train) print('Selected features:', sfs1.k_feature_idx_) Selected features: (1, 2, 3) # Generate the new subsets based on the selected features # Note that the transform call is equivalent to # X_train[:, sfs1.k_feature_idx_] X_train_sfs = sfs1.transform(X_train) X_test_sfs = sfs1.transform(X_test) # Fit the estimator using the new feature subset # and make a prediction on the test data knn.fit(X_train_sfs, y_train) y_pred = knn.predict(X_test_sfs) # Compute the accuracy of the prediction acc = float((y_test == y_pred).sum()) / y_pred.shape[0] print('Test set accuracy: %.2f %%' % (acc * 100)) Test set accuracy: 96.00 % Example 8 -- Sequential Feature Selection and GridSearch In the following example, we are tuning the SFS's estimator using GridSearch. To avoid unwanted behavior or side-effects, it's recommended to use the estimator inside and outside of SFS as separate instances. # Initialize the dataset from sklearn.neighbors import KNeighborsClassifier from sklearn.datasets import load_iris from sklearn.model_selection import train_test_split iris = load_iris() X, y = iris.data, iris.target X_train, X_test, y_train, y_test = train_test_split( X, y, test_size=0.2, random_state=123) from sklearn.model_selection import GridSearchCV from sklearn.pipeline import Pipeline from mlxtend.feature_selection import SequentialFeatureSelector as SFS import mlxtend knn1 = KNeighborsClassifier() knn2 = KNeighborsClassifier() sfs1 = SFS(estimator=knn1, k_features=3, forward=True, floating=False, scoring='accuracy', cv=5) pipe = Pipeline([('sfs', sfs1), ('knn2', knn2)]) param_grid = { 'sfs__k_features': [1, 2, 3], 'sfs__estimator__n_neighbors': [3, 4, 7], # inner knn 'knn2__n_neighbors': [3, 4, 7] # outer knn } gs = GridSearchCV(estimator=pipe, param_grid=param_grid, scoring='accuracy', n_jobs=1, cv=5, refit=False) # run gridearch gs = gs.fit(X_train, y_train) Let's take a look at the suggested hyperparameters below: for i in range(len(gs.cv_results_['params'])): print(gs.cv_results_['params'][i], 'test acc.:', gs.cv_results_['mean_test_score'][i]) The \"best\" parameters determined by GridSearch are ... print(\"Best parameters via GridSearch\", gs.best_params_) Best parameters via GridSearch {'knn2__n_neighbors': 7, 'sfs__estimator__n_neighbors': 3, 'sfs__k_features': 3} pipe.set_params(**gs.best_params_).fit(X_train, y_train) #sk-container-id-1 {color: black;background-color: white;}#sk-container-id-1 pre{padding: 0;}#sk-container-id-1 div.sk-toggleable {background-color: white;}#sk-container-id-1 label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.3em;box-sizing: border-box;text-align: center;}#sk-container-id-1 label.sk-toggleable__label-arrow:before {content: \"\u25b8\";float: left;margin-right: 0.25em;color: #696969;}#sk-container-id-1 label.sk-toggleable__label-arrow:hover:before {color: black;}#sk-container-id-1 div.sk-estimator:hover label.sk-toggleable__label-arrow:before {color: black;}#sk-container-id-1 div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}#sk-container-id-1 div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}#sk-container-id-1 input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}#sk-container-id-1 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {content: \"\u25be\";}#sk-container-id-1 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}#sk-container-id-1 div.sk-estimator {font-family: monospace;background-color: #f0f8ff;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;margin-bottom: 0.5em;}#sk-container-id-1 div.sk-estimator:hover {background-color: #d4ebff;}#sk-container-id-1 div.sk-parallel-item::after {content: \"\";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}#sk-container-id-1 div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 div.sk-serial::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: 0;}#sk-container-id-1 div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;padding-right: 0.2em;padding-left: 0.2em;position: relative;}#sk-container-id-1 div.sk-item {position: relative;z-index: 1;}#sk-container-id-1 div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;position: relative;}#sk-container-id-1 div.sk-item::before, #sk-container-id-1 div.sk-parallel-item::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: -1;}#sk-container-id-1 div.sk-parallel-item {display: flex;flex-direction: column;z-index: 1;position: relative;background-color: white;}#sk-container-id-1 div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}#sk-container-id-1 div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}#sk-container-id-1 div.sk-parallel-item:only-child::after {width: 0;}#sk-container-id-1 div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0 0.4em 0.5em 0.4em;box-sizing: border-box;padding-bottom: 0.4em;background-color: white;}#sk-container-id-1 div.sk-label label {font-family: monospace;font-weight: bold;display: inline-block;line-height: 1.2em;}#sk-container-id-1 div.sk-label-container {text-align: center;}#sk-container-id-1 div.sk-container {/* jupyter's `normalize.less` sets `[hidden] { display: none; }` but bootstrap.min.css set `[hidden] { display: none !important; }` so we also need the `!important` here to be able to override the default hidden behavior on the sphinx rendered scikit-learn.org. See: https://github.com/scikit-learn/scikit-learn/issues/21755 */display: inline-block !important;position: relative;}#sk-container-id-1 div.sk-text-repr-fallback {display: none;} Pipeline(steps=[('sfs', SequentialFeatureSelector(estimator=KNeighborsClassifier(n_neighbors=3), k_features=(3, 3), scoring='accuracy')), ('knn2', KNeighborsClassifier(n_neighbors=7))]) In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org. Pipeline Pipeline(steps=[('sfs', SequentialFeatureSelector(estimator=KNeighborsClassifier(n_neighbors=3), k_features=(3, 3), scoring='accuracy')), ('knn2', KNeighborsClassifier(n_neighbors=7))]) sfs: SequentialFeatureSelector SequentialFeatureSelector(estimator=KNeighborsClassifier(n_neighbors=3), k_features=(3, 3), scoring='accuracy') estimator: KNeighborsClassifier KNeighborsClassifier(n_neighbors=3) KNeighborsClassifier KNeighborsClassifier(n_neighbors=3) KNeighborsClassifier KNeighborsClassifier(n_neighbors=7) Example 9 -- Selecting the \"best\" feature combination in a k-range If k_features is set to to a tuple (min_k, max_k) (new in 0.4.2), the SFS will now select the best feature combination that it discovered by iterating from k=1 to max_k (forward), or max_k to min_k (backward). The size of the returned feature subset is then within max_k to min_k , depending on which combination scored best during cross validation. X.shape (150, 4) from mlxtend.feature_selection import SequentialFeatureSelector as SFS from sklearn.neighbors import KNeighborsClassifier from mlxtend.data import wine_data from sklearn.model_selection import train_test_split from sklearn.preprocessing import StandardScaler from sklearn.pipeline import make_pipeline X, y = wine_data() X_train, X_test, y_train, y_test= train_test_split(X, y, stratify=y, test_size=0.3, random_state=1) knn = KNeighborsClassifier(n_neighbors=2) sfs1 = SFS(estimator=knn, k_features=(3, 10), forward=True, floating=False, scoring='accuracy', cv=5) pipe = make_pipeline(StandardScaler(), sfs1) pipe.fit(X_train, y_train) print('best combination (ACC: %.3f): %s\\n' % (sfs1.k_score_, sfs1.k_feature_idx_)) print('all subsets:\\n', sfs1.subsets_) plot_sfs(sfs1.get_metric_dict(), kind='std_err'); best combination (ACC: 0.992): (0, 1, 2, 3, 6, 8, 9, 10, 11, 12) all subsets: {1: {'feature_idx': (6,), 'cv_scores': array([0.84 , 0.64 , 0.84 , 0.8 , 0.875]), 'avg_score': 0.799, 'feature_names': ('6',)}, 2: {'feature_idx': (6, 9), 'cv_scores': array([0.92 , 0.88 , 1. , 0.96 , 0.91666667]), 'avg_score': 0.9353333333333333, 'feature_names': ('6', '9')}, 3: {'feature_idx': (6, 9, 12), 'cv_scores': array([0.92 , 0.92 , 0.96 , 1. , 0.95833333]), 'avg_score': 0.9516666666666665, 'feature_names': ('6', '9', '12')}, 4: {'feature_idx': (3, 6, 9, 12), 'cv_scores': array([0.96 , 0.96 , 0.96 , 1. , 0.95833333]), 'avg_score': 0.9676666666666666, 'feature_names': ('3', '6', '9', '12')}, 5: {'feature_idx': (3, 6, 9, 10, 12), 'cv_scores': array([0.92, 0.96, 1. , 1. , 1. ]), 'avg_score': 0.976, 'feature_names': ('3', '6', '9', '10', '12')}, 6: {'feature_idx': (2, 3, 6, 9, 10, 12), 'cv_scores': array([0.92, 0.96, 1. , 0.96, 1. ]), 'avg_score': 0.968, 'feature_names': ('2', '3', '6', '9', '10', '12')}, 7: {'feature_idx': (0, 2, 3, 6, 9, 10, 12), 'cv_scores': array([0.92, 0.92, 1. , 1. , 1. ]), 'avg_score': 0.968, 'feature_names': ('0', '2', '3', '6', '9', '10', '12')}, 8: {'feature_idx': (0, 2, 3, 6, 8, 9, 10, 12), 'cv_scores': array([1. , 0.92, 1. , 1. , 1. ]), 'avg_score': 0.984, 'feature_names': ('0', '2', '3', '6', '8', '9', '10', '12')}, 9: {'feature_idx': (0, 2, 3, 6, 8, 9, 10, 11, 12), 'cv_scores': array([1. , 0.92, 1. , 1. , 1. ]), 'avg_score': 0.984, 'feature_names': ('0', '2', '3', '6', '8', '9', '10', '11', '12')}, 10: {'feature_idx': (0, 1, 2, 3, 6, 8, 9, 10, 11, 12), 'cv_scores': array([1. , 0.96, 1. , 1. , 1. ]), 'avg_score': 0.992, 'feature_names': ('0', '1', '2', '3', '6', '8', '9', '10', '11', '12')}} Example 10 -- Using other cross-validation schemes In addition to standard k-fold and stratified k-fold, other cross validation schemes can be used with SequentialFeatureSelector . For example, GroupKFold or LeaveOneOut cross-validation from scikit-learn. Using GroupKFold with SequentialFeatureSelector from mlxtend.feature_selection import SequentialFeatureSelector as SFS from sklearn.neighbors import KNeighborsClassifier from mlxtend.data import iris_data from sklearn.model_selection import GroupKFold import numpy as np X, y = iris_data() groups = np.arange(len(y)) // 10 print('groups: {}'.format(groups)) groups: [ 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 2 2 2 2 2 2 2 2 2 2 3 3 3 3 3 3 3 3 3 3 4 4 4 4 4 4 4 4 4 4 5 5 5 5 5 5 5 5 5 5 6 6 6 6 6 6 6 6 6 6 7 7 7 7 7 7 7 7 7 7 8 8 8 8 8 8 8 8 8 8 9 9 9 9 9 9 9 9 9 9 10 10 10 10 10 10 10 10 10 10 11 11 11 11 11 11 11 11 11 11 12 12 12 12 12 12 12 12 12 12 13 13 13 13 13 13 13 13 13 13 14 14 14 14 14 14 14 14 14 14] Calling the split() method of a scikit-learn cross-validator object will return a generator that yields train, test splits. cv_gen = GroupKFold(4).split(X, y, groups) cv_gen <generator object _BaseKFold.split at 0x17c109580> The cv parameter of SequentialFeatureSelector must be either an int or an iterable yielding train, test splits. This iterable can be constructed by passing the train, test split generator to the built-in list() function. cv = list(cv_gen) knn = KNeighborsClassifier(n_neighbors=2) sfs = SFS(estimator=knn, k_features=2, scoring='accuracy', cv=cv) sfs.fit(X, y) print('best combination (ACC: %.3f): %s\\n' % (sfs.k_score_, sfs.k_feature_idx_)) best combination (ACC: 0.940): (2, 3) Example 11 - Interrupting Long Runs for Intermediate Results If your run is taking too long, it is possible to trigger a KeyboardInterrupt (e.g., ctrl+c on a Mac, or interrupting the cell in a Jupyter notebook) to obtain temporary results. Toy dataset from sklearn.datasets import make_classification from sklearn.model_selection import train_test_split X, y = make_classification( n_samples=20000, n_features=500, n_informative=10, n_redundant=40, n_repeated=25, n_clusters_per_class=5, flip_y=0.05, class_sep=0.5, random_state=123, ) X_train, X_test, y_train, y_test = train_test_split( X, y, test_size=0.2, random_state=123 ) Long run with interruption from mlxtend.feature_selection import SequentialFeatureSelector as SFS from sklearn.linear_model import LogisticRegression model = LogisticRegression() sfs1 = SFS(model, k_features=10, forward=True, floating=False, verbose=2, scoring='accuracy', cv=5) sfs1 = sfs1.fit(X_train, y_train) [Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers. [Parallel(n_jobs=1)]: Done 1 out of 1 | elapsed: 0.0s remaining: 0.0s [Parallel(n_jobs=1)]: Done 500 out of 500 | elapsed: 8.3s finished [2023-05-17 08:36:32] Features: 1/10 -- score: 0.5965[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers. [Parallel(n_jobs=1)]: Done 1 out of 1 | elapsed: 0.0s remaining: 0.0s [Parallel(n_jobs=1)]: Done 499 out of 499 | elapsed: 13.8s finished [2023-05-17 08:36:45] Features: 2/10 -- score: 0.6256875000000001[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers. [Parallel(n_jobs=1)]: Done 1 out of 1 | elapsed: 0.0s remaining: 0.0s [Parallel(n_jobs=1)]: Done 498 out of 498 | elapsed: 18.1s finished [2023-05-17 08:37:03] Features: 3/10 -- score: 0.642[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers. [Parallel(n_jobs=1)]: Done 1 out of 1 | elapsed: 0.0s remaining: 0.0s [Parallel(n_jobs=1)]: Done 497 out of 497 | elapsed: 20.4s finished [2023-05-17 08:37:24] Features: 4/10 -- score: 0.6463125[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers. [Parallel(n_jobs=1)]: Done 1 out of 1 | elapsed: 0.0s remaining: 0.0s [Parallel(n_jobs=1)]: Done 496 out of 496 | elapsed: 22.2s finished [2023-05-17 08:37:46] Features: 5/10 -- score: 0.6495000000000001[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers. [Parallel(n_jobs=1)]: Done 1 out of 1 | elapsed: 0.1s remaining: 0.0s [Parallel(n_jobs=1)]: Done 495 out of 495 | elapsed: 26.1s finished [2023-05-17 08:38:12] Features: 6/10 -- score: 0.6514374999999999[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers. [Parallel(n_jobs=1)]: Done 1 out of 1 | elapsed: 0.0s remaining: 0.0s [Parallel(n_jobs=1)]: Done 494 out of 494 | elapsed: 26.1s finished [2023-05-17 08:38:38] Features: 7/10 -- score: 0.6533749999999999[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers. [Parallel(n_jobs=1)]: Done 1 out of 1 | elapsed: 0.0s remaining: 0.0s [Parallel(n_jobs=1)]: Done 493 out of 493 | elapsed: 25.3s finished [2023-05-17 08:39:04] Features: 8/10 -- score: 0.6545624999999999[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers. [Parallel(n_jobs=1)]: Done 1 out of 1 | elapsed: 0.1s remaining: 0.0s [Parallel(n_jobs=1)]: Done 492 out of 492 | elapsed: 26.3s finished [2023-05-17 08:39:30] Features: 9/10 -- score: 0.6549375[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers. [Parallel(n_jobs=1)]: Done 1 out of 1 | elapsed: 0.1s remaining: 0.0s [Parallel(n_jobs=1)]: Done 491 out of 491 | elapsed: 27.0s finished [2023-05-17 08:39:57] Features: 10/10 -- score: 0.6554374999999999 Finalizing the fit Note that the feature selection run hasn't finished, so certain attributes may not be available. In order to use the SFS instance, it is recommended to call finalize_fit , which will make SFS estimator appear as \"fitted\" process the temporary results: sfs1.finalize_fit() print(sfs1.k_feature_idx_) print(sfs1.k_score_) (30, 128, 144, 160, 184, 229, 256, 356, 439, 458) 0.6554374999999999 Example 12 - Using Pandas DataFrames Optionally, we can also use pandas DataFrames and pandas Series as input to the fit function. In this case, the column names of the pandas DataFrame will be used as feature names. However, note that if custom_feature_names are provided in the fit function, these custom_feature_names take precedence over the DataFrame column-based feature names. import pandas as pd from sklearn.neighbors import KNeighborsClassifier from sklearn.datasets import load_iris from mlxtend.feature_selection import SequentialFeatureSelector as SFS iris = load_iris() X = iris.data y = iris.target knn = KNeighborsClassifier(n_neighbors=4) sfs1 = SFS(knn, k_features=3, forward=True, floating=False, scoring='accuracy', cv=0) X_df = pd.DataFrame(X, columns=['sepal len', 'petal len', 'sepal width', 'petal width']) X_df.head() .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } sepal len petal len sepal width petal width 0 5.1 3.5 1.4 0.2 1 4.9 3.0 1.4 0.2 2 4.7 3.2 1.3 0.2 3 4.6 3.1 1.5 0.2 4 5.0 3.6 1.4 0.2 Also, the target array, y , can be optionally be cast as a Series: y_series = pd.Series(y) y_series.head() 0 0 1 0 2 0 3 0 4 0 dtype: int64 sfs1 = sfs1.fit(X_df, y_series) Note that the only difference of passing a pandas DataFrame as input is that the sfs1.subsets_ array will now contain a new column, sfs1.subsets_ {1: {'feature_idx': (3,), 'cv_scores': array([0.96]), 'avg_score': 0.96, 'feature_names': ('petal width',)}, 2: {'feature_idx': (2, 3), 'cv_scores': array([0.97333333]), 'avg_score': 0.9733333333333334, 'feature_names': ('sepal width', 'petal width')}, 3: {'feature_idx': (1, 2, 3), 'cv_scores': array([0.97333333]), 'avg_score': 0.9733333333333334, 'feature_names': ('petal len', 'sepal width', 'petal width')}} In mlxtend version >= 0.13 pandas DataFrames are supported as feature inputs to the SequentianFeatureSelector instead of NumPy arrays or other NumPy-like array types. Example 13 - Specifying Fixed Feature Sets Often, it may be useful to specify a fixed set of features we want to use for a given model (e.g., determined by prior knowledge or domain knowledge). Since MLxtend v 0.18.0, it is now possible to specify such features via the fixed_features attribute. This will mean that these features are guaranteed to be included in the selected subsets. Note that this feature works for all options regarding forward and backward selection, and using floating selection or not. The example below illustrates how we can set the features 0 and 2 in the dataset as fixed: from sklearn.neighbors import KNeighborsClassifier from sklearn.datasets import load_iris iris = load_iris() X = iris.data y = iris.target knn = KNeighborsClassifier(n_neighbors=3) from mlxtend.feature_selection import SequentialFeatureSelector as SFS sfs1 = SFS(knn, k_features=4, forward=True, floating=False, verbose=2, scoring='accuracy', fixed_features=(0, 2), cv=3) sfs1 = sfs1.fit(X, y) [Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers. [Parallel(n_jobs=1)]: Done 1 out of 1 | elapsed: 0.0s remaining: 0.0s [Parallel(n_jobs=1)]: Done 2 out of 2 | elapsed: 0.0s finished [2023-05-17 08:39:57] Features: 3/4 -- score: 0.9733333333333333[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers. [Parallel(n_jobs=1)]: Done 1 out of 1 | elapsed: 0.0s remaining: 0.0s [Parallel(n_jobs=1)]: Done 1 out of 1 | elapsed: 0.0s finished [2023-05-17 08:39:57] Features: 4/4 -- score: 0.9733333333333333 sfs1.subsets_ {2: {'feature_idx': (0, 2), 'cv_scores': array([0.98, 0.92, 0.94]), 'avg_score': 0.9466666666666667, 'feature_names': ('0', '2')}, 3: {'feature_idx': (0, 2, 3), 'cv_scores': array([0.98, 0.96, 0.98]), 'avg_score': 0.9733333333333333, 'feature_names': ('0', '2', '3')}, 4: {'feature_idx': (0, 1, 2, 3), 'cv_scores': array([0.98, 0.96, 0.98]), 'avg_score': 0.9733333333333333, 'feature_names': ('0', '1', '2', '3')}} If the input dataset is a pandas DataFrame, we can also use the column names directly: import pandas as pd X_df = pd.DataFrame(X, columns=['sepal len', 'petal len', 'sepal width', 'petal width']) X_df.head() .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } sepal len petal len sepal width petal width 0 5.1 3.5 1.4 0.2 1 4.9 3.0 1.4 0.2 2 4.7 3.2 1.3 0.2 3 4.6 3.1 1.5 0.2 4 5.0 3.6 1.4 0.2 sfs2 = SFS(knn, k_features=4, forward=True, floating=False, verbose=2, scoring='accuracy', fixed_features=('sepal len', 'petal len'), cv=3) sfs2 = sfs2.fit(X_df, y_series) [Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers. [Parallel(n_jobs=1)]: Done 1 out of 1 | elapsed: 0.0s remaining: 0.0s [Parallel(n_jobs=1)]: Done 2 out of 2 | elapsed: 0.0s finished [2023-05-17 08:39:57] Features: 3/4 -- score: 0.9466666666666667[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers. [Parallel(n_jobs=1)]: Done 1 out of 1 | elapsed: 0.0s remaining: 0.0s [Parallel(n_jobs=1)]: Done 1 out of 1 | elapsed: 0.0s finished [2023-05-17 08:39:57] Features: 4/4 -- score: 0.9733333333333333 sfs2.subsets_ {2: {'feature_idx': (0, 1), 'cv_scores': array([0.72, 0.74, 0.78]), 'avg_score': 0.7466666666666667, 'feature_names': ('sepal len', 'petal len')}, 3: {'feature_idx': (0, 1, 2), 'cv_scores': array([0.98, 0.92, 0.94]), 'avg_score': 0.9466666666666667, 'feature_names': ('sepal len', 'petal len', 'sepal width')}, 4: {'feature_idx': (0, 1, 2, 3), 'cv_scores': array([0.98, 0.96, 0.98]), 'avg_score': 0.9733333333333333, 'feature_names': ('sepal len', 'petal len', 'sepal width', 'petal width')}} Example 13 - Working with Feature Groups Since mlxtend v0.21.0, it is possible to specify feature groups. Feature groups allow you to group certain features together, such that they are always selected as a group. This can be very useful in contexts similar to one-hot encoding -- if you want to treat the one-hot encoded feature as a single feature: In the following example, we specify sepal length and sepal width as a feature group so that they are always selected together: from sklearn.datasets import load_iris import pandas as pd iris = load_iris() X = iris.data y = iris.target X_df = pd.DataFrame(X, columns=['sepal len', 'petal len', 'sepal wid', 'petal wid']) X_df.head() .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } sepal len petal len sepal wid petal wid 0 5.1 3.5 1.4 0.2 1 4.9 3.0 1.4 0.2 2 4.7 3.2 1.3 0.2 3 4.6 3.1 1.5 0.2 4 5.0 3.6 1.4 0.2 from sklearn.neighbors import KNeighborsClassifier from mlxtend.feature_selection import SequentialFeatureSelector as SFS knn = KNeighborsClassifier(n_neighbors=3) sfs1 = SFS(knn, k_features=2, scoring='accuracy', feature_groups=(['sepal len', 'sepal wid'], ['petal len'], ['petal wid']), cv=3) sfs1 = sfs1.fit(X_df, y) sfs1 = SFS(knn, k_features=2, scoring='accuracy', feature_groups=[[0, 2], [1], [3]], cv=3) sfs1 = sfs1.fit(X, y) Example 14 - Multiclass Metrics Certain scoring metrics like ROC AUC are originally designed for binary classification. However, they can also be used for multiclass settings. It is best to consult this scikit-learn metrics table for this. For example, we can use a ROC AUC One-Vs-Rest score via \u2018\"roc_auc_ovr\" as shown below. from sklearn.datasets import make_blobs X, y = make_blobs(n_samples=10, centers=4, n_features=5, random_state=0) from mlxtend.feature_selection import SequentialFeatureSelector as SFS sfs1 = SFS(knn, k_features=3, forward=True, floating=False, verbose=2, scoring='roc_auc_ovr', cv=0) sfs1 = sfs1.fit(X, y) [Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers. [Parallel(n_jobs=1)]: Done 1 out of 1 | elapsed: 0.0s remaining: 0.0s [Parallel(n_jobs=1)]: Done 5 out of 5 | elapsed: 0.0s finished [2023-05-17 08:39:57] Features: 1/3 -- score: 1.0[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers. [Parallel(n_jobs=1)]: Done 1 out of 1 | elapsed: 0.0s remaining: 0.0s [Parallel(n_jobs=1)]: Done 4 out of 4 | elapsed: 0.0s finished [2023-05-17 08:39:57] Features: 2/3 -- score: 1.0[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers. [Parallel(n_jobs=1)]: Done 1 out of 1 | elapsed: 0.0s remaining: 0.0s [Parallel(n_jobs=1)]: Done 3 out of 3 | elapsed: 0.0s finished [2023-05-17 08:39:57] Features: 3/3 -- score: 1.0 API SequentialFeatureSelector(estimator, k_features=1, forward=True, floating=False, verbose=0, scoring=None, cv=5, n_jobs=1, pre_dispatch='2 n_jobs', clone_estimator=True, fixed_features=None, feature_groups=None)* Sequential Feature Selection for Classification and Regression. Parameters estimator : scikit-learn classifier or regressor k_features : int or tuple or str (default: 1) Number of features to select, where k_features < the full feature set. New in 0.4.2: A tuple containing a min and max value can be provided, and the SFS will consider return any feature combination between min and max that scored highest in cross-validation. For example, the tuple (1, 4) will return any combination from 1 up to 4 features instead of a fixed number of features k. New in 0.8.0: A string argument \"best\" or \"parsimonious\". If \"best\" is provided, the feature selector will return the feature subset with the best cross-validation performance. If \"parsimonious\" is provided as an argument, the smallest feature subset that is within one standard error of the cross-validation performance will be selected. forward : bool (default: True) Forward selection if True, backward selection otherwise floating : bool (default: False) Adds a conditional exclusion/inclusion if True. verbose : int (default: 0), level of verbosity to use in logging. If 0, no output, if 1 number of features in current set, if 2 detailed logging i ncluding timestamp and cv scores at step. scoring : str, callable, or None (default: None) If None (default), uses 'accuracy' for sklearn classifiers and 'r2' for sklearn regressors. If str, uses a sklearn scoring metric string identifier, for example {accuracy, f1, precision, recall, roc_auc} for classifiers, {'mean_absolute_error', 'mean_squared_error'/'neg_mean_squared_error', 'median_absolute_error', 'r2'} for regressors. If a callable object or function is provided, it has to be conform with sklearn's signature scorer(estimator, X, y) ; see https://scikit-learn.org/stable/modules/generated/sklearn.metrics.make_scorer.html for more information. cv : int (default: 5) Integer or iterable yielding train, test splits. If cv is an integer and estimator is a classifier (or y consists of integer class labels) stratified k-fold. Otherwise regular k-fold cross-validation is performed. No cross-validation if cv is None, False, or 0. n_jobs : int (default: 1) The number of CPUs to use for evaluating different feature subsets in parallel. -1 means 'all CPUs'. pre_dispatch : int, or string (default: '2*n_jobs') Controls the number of jobs that get dispatched during parallel execution if n_jobs > 1 or n_jobs=-1 . Reducing this number can be useful to avoid an explosion of memory consumption when more jobs get dispatched than CPUs can process. This parameter can be: None, in which case all the jobs are immediately created and spawned. Use this for lightweight and fast-running jobs, to avoid delays due to on-demand spawning of the jobs An int, giving the exact number of total jobs that are spawned A string, giving an expression as a function of n_jobs, as in 2*n_jobs clone_estimator : bool (default: True) Clones estimator if True; works with the original estimator instance if False. Set to False if the estimator doesn't implement scikit-learn's set_params and get_params methods. In addition, it is required to set cv=0, and n_jobs=1. fixed_features : tuple (default: None) If not None , the feature indices provided as a tuple will be regarded as fixed by the feature selector. For example, if fixed_features=(1, 3, 7) , the 2nd, 4th, and 8th feature are guaranteed to be present in the solution. Note that if fixed_features is not None , make sure that the number of features to be selected is greater than len(fixed_features) . In other words, ensure that k_features > len(fixed_features) . New in mlxtend v. 0.18.0. feature_groups : list or None (default: None) Optional argument for treating certain features as a group. This means, the features within a group are always selected together, never split. For example, feature_groups=[[1], [2], [3, 4, 5]] specifies 3 feature groups. In this case, possible feature selection results with k_features=2 are [[1], [2] , [[1], [3, 4, 5]] , or [[2], [3, 4, 5]] . Feature groups can be useful for interpretability, for example, if features 3, 4, 5 are one-hot encoded features. (For more details, please read the notes at the bottom of this docstring). New in mlxtend v. 0.21.0. Attributes k_feature_idx_ : array-like, shape = [n_predictions] Feature Indices of the selected feature subsets. k_feature_names_ : array-like, shape = [n_predictions] Feature names of the selected feature subsets. If pandas DataFrames are used in the fit method, the feature names correspond to the column names. Otherwise, the feature names are string representation of the feature array indices. New in v 0.13.0. k_score_ : float Cross validation average score of the selected subset. subsets_ : dict A dictionary of selected feature subsets during the sequential selection, where the dictionary keys are the lengths k of these feature subsets. If the parameter feature_groups is not None, the value of key indicates the number of groups that are selected together. The dictionary values are dictionaries themselves with the following keys: 'feature_idx' (tuple of indices of the feature subset) 'feature_names' (tuple of feature names of the feat. subset) 'cv_scores' (list individual cross-validation scores) 'avg_score' (average cross-validation score) Note that if pandas DataFrames are used in the fit method, the 'feature_names' correspond to the column names. Otherwise, the feature names are string representation of the feature array indices. The 'feature_names' is new in v 0.13.0. Notes (1) If parameter feature_groups is not None, the number of features is equal to the number of feature groups, i.e. len(feature_groups) . For example, if feature_groups = [[0], [1], [2, 3], [4]] , then the max_features value cannot exceed 4. (2) Although two or more individual features may be considered as one group throughout the feature-selection process, it does not mean the individual features of that group have the same impact on the outcome. For instance, in linear regression, the coefficient of the feature 2 and 3 can be different even if they are considered as one group in feature_groups. (3) If both fixed_features and feature_groups are specified, ensure that each feature group contains the fixed_features selection. E.g., for a 3-feature set fixed_features=[0, 1] and feature_groups=[[0, 1], [2]] is valid; fixed_features=[0, 1] and feature_groups=[[0], [1, 2]] is not valid. (4) In case of KeyboardInterrupt, the dictionary subsets may not be completed. If user is still interested in getting the best score, they can use method `finalize_fit`. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/feature_selection/SequentialFeatureSelector/ Methods finalize_fit() None fit(X, y, groups=None, fit_params) Perform feature selection and learn model from training data. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. New in v 0.13.0: pandas DataFrames are now also accepted as argument for X. y : array-like, shape = [n_samples] Target values. New in v 0.13.0: pandas DataFrames are now also accepted as argument for y. groups : array-like, with shape (n_samples,), optional Group labels for the samples used while splitting the dataset into train/test set. Passed to the fit method of the cross-validator. fit_params : various, optional Additional parameters that are being passed to the estimator. For example, sample_weights=weights . Returns self : object fit_transform(X, y, groups=None, fit_params) Fit to training data then reduce X to its most important features. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. New in v 0.13.0: pandas DataFrames are now also accepted as argument for X. y : array-like, shape = [n_samples] Target values. New in v 0.13.0: a pandas Series are now also accepted as argument for y. groups : array-like, with shape (n_samples,), optional Group labels for the samples used while splitting the dataset into train/test set. Passed to the fit method of the cross-validator. fit_params : various, optional Additional parameters that are being passed to the estimator. For example, sample_weights=weights . Returns Reduced feature subset of X, shape={n_samples, k_features} generate_error_message_k_features(name) None get_metric_dict(confidence_interval=0.95) Return metric dictionary Parameters confidence_interval : float (default: 0.95) A positive float between 0.0 and 1.0 to compute the confidence interval bounds of the CV score averages. Returns Dictionary with items where each dictionary value is a list with the number of iterations (number of feature subsets) as its length. The dictionary keys corresponding to these lists are as follows: 'feature_idx': tuple of the indices of the feature subset 'cv_scores': list with individual CV scores 'avg_score': of CV average scores 'std_dev': standard deviation of the CV score average 'std_err': standard error of the CV score average 'ci_bound': confidence interval bound of the CV score average get_params(deep=True) Get parameters for this estimator. Parameters deep : bool, default=True If True, will return the parameters for this estimator and contained subobjects that are estimators. Returns params : dict Parameter names mapped to their values. set_params( params) Set the parameters of this estimator. Valid parameter keys can be listed with get_params() . Returns self transform(X) Reduce X to its most important features. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. New in v 0.13.0: pandas DataFrames are now also accepted as argument for X. Returns Reduced feature subset of X, shape={n_samples, k_features} Properties named_estimators Returns List of named estimator tuples, like [('svc', SVC(...))] ython","title":"SequentialFeatureSelector: The popular forward and backward feature selection approaches (including floating variants)"},{"location":"user_guide/feature_selection/SequentialFeatureSelector/#sequentialfeatureselector-the-popular-forward-and-backward-feature-selection-approaches-including-floating-variants","text":"Implementation of sequential feature algorithms (SFAs) -- greedy search algorithms -- that have been developed as a suboptimal solution to the computationally often not feasible exhaustive search. from mlxtend.feature_selection import SequentialFeatureSelector","title":"SequentialFeatureSelector: The popular forward and backward feature selection approaches (including floating variants)"},{"location":"user_guide/feature_selection/SequentialFeatureSelector/#overview","text":"Sequential feature selection algorithms are a family of greedy search algorithms that are used to reduce an initial d -dimensional feature space to a k -dimensional feature subspace where k < d . The motivation behind feature selection algorithms is to automatically select a subset of features most relevant to the problem. The goal of feature selection is two-fold: We want to improve the computational efficiency and reduce the model's generalization error by removing irrelevant features or noise. In addition, a wrapper approach such as sequential feature selection is advantageous if embedded feature selection -- for example, a regularization penalty like LASSO -- is not applicable. In a nutshell, SFAs remove or add one feature at a time based on the classifier performance until a feature subset of the desired size k is reached. There are four different flavors of SFAs available via the SequentialFeatureSelector : Sequential Forward Selection (SFS) Sequential Backward Selection (SBS) Sequential Forward Floating Selection (SFFS) Sequential Backward Floating Selection (SBFS) The floating variants, SFFS and SBFS, can be considered extensions to the simpler SFS and SBS algorithms. The floating algorithms have an additional exclusion or inclusion step to remove features once they were included (or excluded) so that a larger number of feature subset combinations can be sampled. It is important to emphasize that this step is conditional and only occurs if the resulting feature subset is assessed as \"better\" by the criterion function after the removal (or addition) of a particular feature. Furthermore, I added an optional check to skip the conditional exclusion steps if the algorithm gets stuck in cycles. How is this different from Recursive Feature Elimination (RFE) -- e.g., as implemented in sklearn.feature_selection.RFE ? RFE is computationally less complex using the feature weight coefficients (e.g., linear models) or feature importance (tree-based algorithms) to eliminate features recursively, whereas SFSs eliminate (or add) features based on a user-defined classifier/regression performance metric.","title":"Overview"},{"location":"user_guide/feature_selection/SequentialFeatureSelector/#tutorial-videos","text":"","title":"Tutorial Videos"},{"location":"user_guide/feature_selection/SequentialFeatureSelector/#visual-illustration","text":"A visual illustration of the sequential backward selection process is provided below, from the paper Joe Bemister-Buffington, Alex J. Wolf, Sebastian Raschka, and Leslie A. Kuhn (2020) Machine Learning to Identify Flexibility Signatures of Class A GPCR Inhibition Biomolecules 2020, 10, 454. https://www.mdpi.com/2218-273X/10/3/454#","title":"Visual Illustration"},{"location":"user_guide/feature_selection/SequentialFeatureSelector/#algorithmic-details","text":"","title":"Algorithmic Details"},{"location":"user_guide/feature_selection/SequentialFeatureSelector/#sequential-forward-selection-sfs","text":"Input: Y = \\{y_1, y_2, ..., y_d\\} The SFS algorithm takes the whole d -dimensional feature set as input. Output: X_k = \\{x_j \\; | \\;j = 1, 2, ..., k; \\; x_j \\in Y\\} , where k = (0, 1, 2, ..., d) SFS returns a subset of features; the number of selected features k , where k < d , has to be specified a priori . Initialization: X_0 = \\emptyset , k = 0 We initialize the algorithm with an empty set \\emptyset (\"null set\") so that k = 0 (where k is the size of the subset). Step 1 (Inclusion): x^+ = \\text{ arg max } J(X_k + x), \\text{ where } x \\in Y - X_k X_{k+1} = X_k + x^+ k = k + 1 Go to Step 1 in this step, we add an additional feature, x^+ , to our feature subset X_k . x^+ is the feature that maximizes our criterion function, that is, the feature that is associated with the best classifier performance if it is added to X_k . We repeat this procedure until the termination criterion is satisfied. Termination: k = p We add features from the feature subset X_k until the feature subset of size k contains the number of desired features p that we specified a priori .","title":"Sequential Forward Selection (SFS)"},{"location":"user_guide/feature_selection/SequentialFeatureSelector/#sequential-backward-selection-sbs","text":"Input: the set of all features, Y = \\{y_1, y_2, ..., y_d\\} The SBS algorithm takes the whole feature set as input. Output: X_k = \\{x_j \\; | \\;j = 1, 2, ..., k; \\; x_j \\in Y\\} , where k = (0, 1, 2, ..., d) SBS returns a subset of features; the number of selected features k , where k < d , has to be specified a priori . Initialization: X_0 = Y , k = d We initialize the algorithm with the given feature set so that the k = d . Step 1 (Exclusion): x^- = \\text{ arg max } J(X_k - x), \\text{ where } x \\in X_k X_{k-1} = X_k - x^- k = k - 1 Go to Step 1 In this step, we remove a feature, x^- from our feature subset X_k . x^- is the feature that maximizes our criterion function upon re,oval, that is, the feature that is associated with the best classifier performance if it is removed from X_k . We repeat this procedure until the termination criterion is satisfied. Termination: k = p We add features from the feature subset X_k until the feature subset of size k contains the number of desired features p that we specified a priori .","title":"Sequential Backward Selection (SBS)"},{"location":"user_guide/feature_selection/SequentialFeatureSelector/#sequential-backward-floating-selection-sbfs","text":"Input: the set of all features, Y = \\{y_1, y_2, ..., y_d\\} The SBFS algorithm takes the whole feature set as input. Output: X_k = \\{x_j \\; | \\;j = 1, 2, ..., k; \\; x_j \\in Y\\} , where k = (0, 1, 2, ..., d) SBFS returns a subset of features; the number of selected features k , where k < d , has to be specified a priori . Initialization: X_0 = Y , k = d We initialize the algorithm with the given feature set so that the k = d . Step 1 (Exclusion): x^- = \\text{ arg max } J(X_k - x), \\text{ where } x \\in X_k X_{k-1} = X_k - x^- k = k - 1 Go to Step 2 In this step, we remove a feature, x^- from our feature subset X_k . x^- is the feature that maximizes our criterion function upon removal, that is, the feature that is associated with the best classifier performance if it is removed from X_k . Step 2 (Conditional Inclusion): x^+ = \\text{ arg max } J(X_k + x), \\text{ where } x \\in Y - X_k if J(X_k + x) > J(X_k) : X_{k+1} = X_k + x^+ k = k + 1 Go to Step 1 In Step 2, we search for features that improve the classifier performance if they are added back to the feature subset. If such features exist, we add the feature x^+ for which the performance improvement is maximized. If k = 2 or an improvement cannot be made (i.e., such feature x^+ cannot be found), go back to step 1; else, repeat this step. Termination: k = p We add features from the feature subset X_k until the feature subset of size k contains the number of desired features p that we specified a priori .","title":"Sequential Backward Floating Selection (SBFS)"},{"location":"user_guide/feature_selection/SequentialFeatureSelector/#sequential-forward-floating-selection-sffs","text":"Input: the set of all features, Y = \\{y_1, y_2, ..., y_d\\} The SFFS algorithm takes the whole feature set as input, if our feature space consists of, e.g. 10, if our feature space consists of 10 dimensions ( d = 10 ). Output: a subset of features, X_k = \\{x_j \\; | \\;j = 1, 2, ..., k; \\; x_j \\in Y\\} , where k = (0, 1, 2, ..., d) The returned output of the algorithm is a subset of the feature space of a specified size. E.g., a subset of 5 features from a 10-dimensional feature space ( k = 5, d = 10 ). Initialization: X_0 = \\emptyset , k = 0 We initialize the algorithm with an empty set (\"null set\") so that the k = 0 (where k is the size of the subset) Step 1 (Inclusion): x^+ = \\text{ arg max } J(X_k + x), \\text{ where } x \\in Y - X_k X_{k+1} = X_k + x^+ k = k + 1 Go to Step 2 Step 2 (Conditional Exclusion): x^- = \\text{ arg max } J(X_k - x), \\text{ where } x \\in X_k if \\; J(X_k - x) > J(X_k) : X_{k-1} = X_k - x^- k = k - 1 Go to Step 1 In step 1, we include the feature from the feature space that leads to the best performance increase for our feature subset (assessed by the criterion function ). Then, we go over to step 2 In step 2, we only remove a feature if the resulting subset would gain an increase in performance. If k = 2 or an improvement cannot be made (i.e., such feature x^+ cannot be found), go back to step 1; else, repeat this step. Steps 1 and 2 are repeated until the Termination criterion is reached. Termination: stop when k equals the number of desired features","title":"Sequential Forward Floating Selection (SFFS)"},{"location":"user_guide/feature_selection/SequentialFeatureSelector/#references","text":"Ferri, F. J., Pudil P., Hatef, M., Kittler, J. (1994). \"Comparative study of techniques for large-scale feature selection.\" Pattern Recognition in Practice IV : 403-413. Pudil, P., Novovi\u010dov\u00e1, J., & Kittler, J. (1994). \"Floating search methods in feature selection.\" Pattern recognition letters 15.11 (1994): 1119-1125.","title":"References"},{"location":"user_guide/feature_selection/SequentialFeatureSelector/#example-1-a-simple-sequential-forward-selection-example","text":"Initializing a simple classifier from scikit-learn: from sklearn.neighbors import KNeighborsClassifier from sklearn.datasets import load_iris iris = load_iris() X = iris.data y = iris.target knn = KNeighborsClassifier(n_neighbors=4) We start by selection the \"best\" 3 features from the Iris dataset via Sequential Forward Selection (SFS). Here, we set forward=True and floating=False . By choosing cv=0 , we don't perform any cross-validation, therefore, the performance (here: 'accuracy' ) is computed entirely on the training set. from mlxtend.feature_selection import SequentialFeatureSelector as SFS sfs1 = SFS(knn, k_features=3, forward=True, floating=False, verbose=2, scoring='accuracy', cv=0) sfs1 = sfs1.fit(X, y) [Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers. [Parallel(n_jobs=1)]: Done 1 out of 1 | elapsed: 0.0s remaining: 0.0s [Parallel(n_jobs=1)]: Done 4 out of 4 | elapsed: 0.0s finished [2023-05-17 08:36:17] Features: 1/3 -- score: 0.96[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers. [Parallel(n_jobs=1)]: Done 1 out of 1 | elapsed: 0.0s remaining: 0.0s [Parallel(n_jobs=1)]: Done 3 out of 3 | elapsed: 0.0s finished [2023-05-17 08:36:17] Features: 2/3 -- score: 0.9733333333333334[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers. [Parallel(n_jobs=1)]: Done 1 out of 1 | elapsed: 0.0s remaining: 0.0s [Parallel(n_jobs=1)]: Done 2 out of 2 | elapsed: 0.0s finished [2023-05-17 08:36:17] Features: 3/3 -- score: 0.9733333333333334 Via the subsets_ attribute, we can take a look at the selected feature indices at each step: sfs1.subsets_ {1: {'feature_idx': (3,), 'cv_scores': array([0.96]), 'avg_score': 0.96, 'feature_names': ('3',)}, 2: {'feature_idx': (2, 3), 'cv_scores': array([0.97333333]), 'avg_score': 0.9733333333333334, 'feature_names': ('2', '3')}, 3: {'feature_idx': (1, 2, 3), 'cv_scores': array([0.97333333]), 'avg_score': 0.9733333333333334, 'feature_names': ('1', '2', '3')}} sfs1 = sfs1.fit(X, y) sfs1.subsets_ [Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers. [Parallel(n_jobs=1)]: Done 1 out of 1 | elapsed: 0.0s remaining: 0.0s [Parallel(n_jobs=1)]: Done 4 out of 4 | elapsed: 0.0s finished [2023-05-17 08:36:17] Features: 1/3 -- score: 0.96[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers. [Parallel(n_jobs=1)]: Done 1 out of 1 | elapsed: 0.0s remaining: 0.0s [Parallel(n_jobs=1)]: Done 3 out of 3 | elapsed: 0.0s finished [2023-05-17 08:36:17] Features: 2/3 -- score: 0.9733333333333334[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers. [Parallel(n_jobs=1)]: Done 1 out of 1 | elapsed: 0.0s remaining: 0.0s [Parallel(n_jobs=1)]: Done 2 out of 2 | elapsed: 0.0s finished [2023-05-17 08:36:17] Features: 3/3 -- score: 0.9733333333333334 {1: {'feature_idx': (3,), 'cv_scores': array([0.96]), 'avg_score': 0.96, 'feature_names': ('3',)}, 2: {'feature_idx': (2, 3), 'cv_scores': array([0.97333333]), 'avg_score': 0.9733333333333334, 'feature_names': ('2', '3')}, 3: {'feature_idx': (1, 2, 3), 'cv_scores': array([0.97333333]), 'avg_score': 0.9733333333333334, 'feature_names': ('1', '2', '3')}} Furthermore, we can access the indices of the 3 best features directly via the k_feature_idx_ attribute: sfs1.k_feature_idx_ (1, 2, 3) Finally, the prediction score for these 3 features can be accesses via k_score_ : sfs1.k_score_ 0.9733333333333334 Feature Names When working with large datasets, the feature indices might be hard to interpret. In this case, we recommend using pandas DataFrames with distinct column names as input: import pandas as pd df_X = pd.DataFrame(X, columns=[\"Sepal length\", \"Sepal width\", \"Petal length\", \"Petal width\"]) df_X.head() .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } Sepal length Sepal width Petal length Petal width 0 5.1 3.5 1.4 0.2 1 4.9 3.0 1.4 0.2 2 4.7 3.2 1.3 0.2 3 4.6 3.1 1.5 0.2 4 5.0 3.6 1.4 0.2 sfs1 = sfs1.fit(df_X, y) print('Best accuracy score: %.2f' % sfs1.k_score_) print('Best subset (indices):', sfs1.k_feature_idx_) print('Best subset (corresponding names):', sfs1.k_feature_names_) [Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers. Best accuracy score: 0.97 Best subset (indices): (1, 2, 3) Best subset (corresponding names): ('Sepal width', 'Petal length', 'Petal width') [Parallel(n_jobs=1)]: Done 1 out of 1 | elapsed: 0.0s remaining: 0.0s [Parallel(n_jobs=1)]: Done 4 out of 4 | elapsed: 0.0s finished [2023-05-17 08:36:17] Features: 1/3 -- score: 0.96[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers. [Parallel(n_jobs=1)]: Done 1 out of 1 | elapsed: 0.0s remaining: 0.0s [Parallel(n_jobs=1)]: Done 3 out of 3 | elapsed: 0.0s finished [2023-05-17 08:36:17] Features: 2/3 -- score: 0.9733333333333334[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers. [Parallel(n_jobs=1)]: Done 1 out of 1 | elapsed: 0.0s remaining: 0.0s [Parallel(n_jobs=1)]: Done 2 out of 2 | elapsed: 0.0s finished [2023-05-17 08:36:17] Features: 3/3 -- score: 0.9733333333333334","title":"Example 1 - A simple Sequential Forward Selection example"},{"location":"user_guide/feature_selection/SequentialFeatureSelector/#example-2-toggling-between-sfs-sbs-sffs-and-sbfs","text":"Using the forward and floating parameters, we can toggle between SFS, SBS, SFFS, and SBFS as shown below. Note that we are performing (stratified) 4-fold cross-validation for more robust estimates in contrast to Example 1. Via n_jobs=-1 , we choose to run the cross-validation on all our available CPU cores. # Sequential Forward Selection sfs = SFS(knn, k_features=3, forward=True, floating=False, scoring='accuracy', cv=4, n_jobs=-1) sfs = sfs.fit(X, y) print('\\nSequential Forward Selection (k=3):') print(sfs.k_feature_idx_) print('CV Score:') print(sfs.k_score_) ################################################### # Sequential Backward Selection sbs = SFS(knn, k_features=3, forward=False, floating=False, scoring='accuracy', cv=4, n_jobs=-1) sbs = sbs.fit(X, y) print('\\nSequential Backward Selection (k=3):') print(sbs.k_feature_idx_) print('CV Score:') print(sbs.k_score_) ################################################### # Sequential Forward Floating Selection sffs = SFS(knn, k_features=3, forward=True, floating=True, scoring='accuracy', cv=4, n_jobs=-1) sffs = sffs.fit(X, y) print('\\nSequential Forward Floating Selection (k=3):') print(sffs.k_feature_idx_) print('CV Score:') print(sffs.k_score_) ################################################### # Sequential Backward Floating Selection sbfs = SFS(knn, k_features=3, forward=False, floating=True, scoring='accuracy', cv=4, n_jobs=-1) sbfs = sbfs.fit(X, y) print('\\nSequential Backward Floating Selection (k=3):') print(sbfs.k_feature_idx_) print('CV Score:') print(sbfs.k_score_) Sequential Forward Selection (k=3): (1, 2, 3) CV Score: 0.9731507823613088 Sequential Backward Selection (k=3): (1, 2, 3) CV Score: 0.9731507823613088 Sequential Forward Floating Selection (k=3): (1, 2, 3) CV Score: 0.9731507823613088 Sequential Backward Floating Selection (k=3): (1, 2, 3) CV Score: 0.9731507823613088 In this simple scenario, selecting the best 3 features out of the 4 available features in the Iris set, we end up with similar results regardless of which sequential selection algorithms we used.","title":"Example 2 - Toggling between SFS, SBS, SFFS, and SBFS"},{"location":"user_guide/feature_selection/SequentialFeatureSelector/#example-3-visualizing-the-results-in-dataframes","text":"For our convenience, we can visualize the output from the feature selection in a pandas DataFrame format using the get_metric_dict method of the SequentialFeatureSelector object. The columns std_dev and std_err represent the standard deviation and standard errors of the cross-validation scores, respectively. Below, we see the DataFrame of the Sequential Forward Selector from Example 2: import pandas as pd pd.DataFrame.from_dict(sfs.get_metric_dict()).T .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } feature_idx cv_scores avg_score feature_names ci_bound std_dev std_err 1 (3,) [0.9736842105263158, 0.9473684210526315, 0.918... 0.959993 (3,) 0.048319 0.030143 0.017403 2 (2, 3) [0.9736842105263158, 0.9473684210526315, 0.918... 0.959993 (2, 3) 0.048319 0.030143 0.017403 3 (1, 2, 3) [0.9736842105263158, 1.0, 0.9459459459459459, ... 0.973151 (1, 2, 3) 0.030639 0.019113 0.011035 Now, let's compare it to the Sequential Backward Selector: pd.DataFrame.from_dict(sbs.get_metric_dict()).T .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } feature_idx cv_scores avg_score feature_names ci_bound std_dev std_err 4 (0, 1, 2, 3) [0.9736842105263158, 0.9473684210526315, 0.918... 0.953236 (0, 1, 2, 3) 0.03602 0.022471 0.012974 3 (1, 2, 3) [0.9736842105263158, 1.0, 0.9459459459459459, ... 0.973151 (1, 2, 3) 0.030639 0.019113 0.011035 We can see that both SFS and SBFS found the same \"best\" 3 features, however, the intermediate steps where obviously different. The ci_bound column in the DataFrames above represents the confidence interval around the computed cross-validation scores. By default, a confidence interval of 95% is used, but we can use different confidence bounds via the confidence_interval parameter. E.g., the confidence bounds for a 90% confidence interval can be obtained as follows: pd.DataFrame.from_dict(sbs.get_metric_dict(confidence_interval=0.90)).T .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } feature_idx cv_scores avg_score feature_names ci_bound std_dev std_err 4 (0, 1, 2, 3) [0.9736842105263158, 0.9473684210526315, 0.918... 0.953236 (0, 1, 2, 3) 0.027658 0.022471 0.012974 3 (1, 2, 3) [0.9736842105263158, 1.0, 0.9459459459459459, ... 0.973151 (1, 2, 3) 0.023525 0.019113 0.011035","title":"Example 3 - Visualizing the results in DataFrames"},{"location":"user_guide/feature_selection/SequentialFeatureSelector/#example-4-plotting-the-results","text":"After importing the little helper function plotting.plot_sequential_feature_selection , we can also visualize the results using matplotlib figures. from mlxtend.plotting import plot_sequential_feature_selection as plot_sfs import matplotlib.pyplot as plt sfs = SFS(knn, k_features=4, forward=True, floating=False, scoring='accuracy', verbose=2, cv=5) sfs = sfs.fit(X, y) fig1 = plot_sfs(sfs.get_metric_dict(), kind='std_dev') plt.ylim([0.8, 1]) plt.title('Sequential Forward Selection (w. StdDev)') plt.grid() plt.show() [Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers. [Parallel(n_jobs=1)]: Done 1 out of 1 | elapsed: 0.0s remaining: 0.0s [Parallel(n_jobs=1)]: Done 4 out of 4 | elapsed: 0.0s finished [2023-05-17 08:36:18] Features: 1/4 -- score: 0.96[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers. [Parallel(n_jobs=1)]: Done 1 out of 1 | elapsed: 0.0s remaining: 0.0s [Parallel(n_jobs=1)]: Done 3 out of 3 | elapsed: 0.0s finished [2023-05-17 08:36:18] Features: 2/4 -- score: 0.9666666666666668[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers. [Parallel(n_jobs=1)]: Done 1 out of 1 | elapsed: 0.0s remaining: 0.0s [Parallel(n_jobs=1)]: Done 2 out of 2 | elapsed: 0.0s finished [2023-05-17 08:36:18] Features: 3/4 -- score: 0.9533333333333334[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers. [Parallel(n_jobs=1)]: Done 1 out of 1 | elapsed: 0.0s remaining: 0.0s [Parallel(n_jobs=1)]: Done 1 out of 1 | elapsed: 0.0s finished [2023-05-17 08:36:18] Features: 4/4 -- score: 0.9733333333333334","title":"Example 4 - Plotting the results"},{"location":"user_guide/feature_selection/SequentialFeatureSelector/#example-5-sequential-feature-selection-for-regression","text":"Similar to the classification examples above, the SequentialFeatureSelector also supports scikit-learn's estimators for regression. from sklearn.linear_model import LinearRegression from sklearn.datasets import fetch_california_housing data = fetch_california_housing() X, y = data.data, data.target lr = LinearRegression() sfs = SFS(lr, k_features=8, forward=True, floating=False, scoring='neg_mean_squared_error', cv=10) sfs = sfs.fit(X, y) fig = plot_sfs(sfs.get_metric_dict(), kind='std_err') plt.title('Sequential Forward Selection (w. StdErr)') plt.grid() plt.show()","title":"Example 5 - Sequential Feature Selection for Regression"},{"location":"user_guide/feature_selection/SequentialFeatureSelector/#example-6-feature-selection-with-fixed-trainvalidation-splits","text":"If you do not wish to use cross-validation (here: k-fold cross-validation, i.e., rotating training and validation folds), you can use the PredefinedHoldoutSplit class to specify your own, fixed training and validation split. from sklearn.datasets import load_iris from mlxtend.evaluate import PredefinedHoldoutSplit import numpy as np iris = load_iris() X = iris.data y = iris.target rng = np.random.RandomState(123) my_validation_indices = rng.permutation(np.arange(150))[:30] print(my_validation_indices) [ 72 112 132 88 37 138 87 42 8 90 141 33 59 116 135 104 36 13 63 45 28 133 24 127 46 20 31 121 117 4] from sklearn.neighbors import KNeighborsClassifier from mlxtend.feature_selection import SequentialFeatureSelector as SFS knn = KNeighborsClassifier(n_neighbors=4) piter = PredefinedHoldoutSplit(my_validation_indices) sfs1 = SFS(knn, k_features=3, forward=True, floating=False, verbose=2, scoring='accuracy', cv=piter) sfs1 = sfs1.fit(X, y) [Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers. [Parallel(n_jobs=1)]: Done 1 out of 1 | elapsed: 0.0s remaining: 0.0s [Parallel(n_jobs=1)]: Done 4 out of 4 | elapsed: 0.0s finished [2023-05-17 08:36:19] Features: 1/3 -- score: 0.9666666666666667[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers. [Parallel(n_jobs=1)]: Done 1 out of 1 | elapsed: 0.0s remaining: 0.0s [Parallel(n_jobs=1)]: Done 3 out of 3 | elapsed: 0.0s finished [2023-05-17 08:36:19] Features: 2/3 -- score: 0.9666666666666667[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers. [Parallel(n_jobs=1)]: Done 1 out of 1 | elapsed: 0.0s remaining: 0.0s [Parallel(n_jobs=1)]: Done 2 out of 2 | elapsed: 0.0s finished [2023-05-17 08:36:19] Features: 3/3 -- score: 0.9666666666666667","title":"Example 6 -- Feature Selection with Fixed Train/Validation Splits"},{"location":"user_guide/feature_selection/SequentialFeatureSelector/#example-7-using-the-selected-feature-subset-for-making-new-predictions","text":"# Initialize the dataset from sklearn.neighbors import KNeighborsClassifier from sklearn.datasets import load_iris from sklearn.model_selection import train_test_split iris = load_iris() X, y = iris.data, iris.target X_train, X_test, y_train, y_test = train_test_split( X, y, test_size=0.33, random_state=1) knn = KNeighborsClassifier(n_neighbors=4) # Select the \"best\" three features via # 5-fold cross-validation on the training set. from mlxtend.feature_selection import SequentialFeatureSelector as SFS sfs1 = SFS(knn, k_features=3, forward=True, floating=False, scoring='accuracy', cv=5) sfs1 = sfs1.fit(X_train, y_train) print('Selected features:', sfs1.k_feature_idx_) Selected features: (1, 2, 3) # Generate the new subsets based on the selected features # Note that the transform call is equivalent to # X_train[:, sfs1.k_feature_idx_] X_train_sfs = sfs1.transform(X_train) X_test_sfs = sfs1.transform(X_test) # Fit the estimator using the new feature subset # and make a prediction on the test data knn.fit(X_train_sfs, y_train) y_pred = knn.predict(X_test_sfs) # Compute the accuracy of the prediction acc = float((y_test == y_pred).sum()) / y_pred.shape[0] print('Test set accuracy: %.2f %%' % (acc * 100)) Test set accuracy: 96.00 %","title":"Example 7 -- Using the Selected Feature Subset For Making New Predictions"},{"location":"user_guide/feature_selection/SequentialFeatureSelector/#example-8-sequential-feature-selection-and-gridsearch","text":"In the following example, we are tuning the SFS's estimator using GridSearch. To avoid unwanted behavior or side-effects, it's recommended to use the estimator inside and outside of SFS as separate instances. # Initialize the dataset from sklearn.neighbors import KNeighborsClassifier from sklearn.datasets import load_iris from sklearn.model_selection import train_test_split iris = load_iris() X, y = iris.data, iris.target X_train, X_test, y_train, y_test = train_test_split( X, y, test_size=0.2, random_state=123) from sklearn.model_selection import GridSearchCV from sklearn.pipeline import Pipeline from mlxtend.feature_selection import SequentialFeatureSelector as SFS import mlxtend knn1 = KNeighborsClassifier() knn2 = KNeighborsClassifier() sfs1 = SFS(estimator=knn1, k_features=3, forward=True, floating=False, scoring='accuracy', cv=5) pipe = Pipeline([('sfs', sfs1), ('knn2', knn2)]) param_grid = { 'sfs__k_features': [1, 2, 3], 'sfs__estimator__n_neighbors': [3, 4, 7], # inner knn 'knn2__n_neighbors': [3, 4, 7] # outer knn } gs = GridSearchCV(estimator=pipe, param_grid=param_grid, scoring='accuracy', n_jobs=1, cv=5, refit=False) # run gridearch gs = gs.fit(X_train, y_train) Let's take a look at the suggested hyperparameters below: for i in range(len(gs.cv_results_['params'])): print(gs.cv_results_['params'][i], 'test acc.:', gs.cv_results_['mean_test_score'][i]) The \"best\" parameters determined by GridSearch are ... print(\"Best parameters via GridSearch\", gs.best_params_) Best parameters via GridSearch {'knn2__n_neighbors': 7, 'sfs__estimator__n_neighbors': 3, 'sfs__k_features': 3} pipe.set_params(**gs.best_params_).fit(X_train, y_train) #sk-container-id-1 {color: black;background-color: white;}#sk-container-id-1 pre{padding: 0;}#sk-container-id-1 div.sk-toggleable {background-color: white;}#sk-container-id-1 label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.3em;box-sizing: border-box;text-align: center;}#sk-container-id-1 label.sk-toggleable__label-arrow:before {content: \"\u25b8\";float: left;margin-right: 0.25em;color: #696969;}#sk-container-id-1 label.sk-toggleable__label-arrow:hover:before {color: black;}#sk-container-id-1 div.sk-estimator:hover label.sk-toggleable__label-arrow:before {color: black;}#sk-container-id-1 div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}#sk-container-id-1 div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}#sk-container-id-1 input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}#sk-container-id-1 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {content: \"\u25be\";}#sk-container-id-1 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}#sk-container-id-1 div.sk-estimator {font-family: monospace;background-color: #f0f8ff;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;margin-bottom: 0.5em;}#sk-container-id-1 div.sk-estimator:hover {background-color: #d4ebff;}#sk-container-id-1 div.sk-parallel-item::after {content: \"\";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}#sk-container-id-1 div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 div.sk-serial::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: 0;}#sk-container-id-1 div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;padding-right: 0.2em;padding-left: 0.2em;position: relative;}#sk-container-id-1 div.sk-item {position: relative;z-index: 1;}#sk-container-id-1 div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;position: relative;}#sk-container-id-1 div.sk-item::before, #sk-container-id-1 div.sk-parallel-item::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: -1;}#sk-container-id-1 div.sk-parallel-item {display: flex;flex-direction: column;z-index: 1;position: relative;background-color: white;}#sk-container-id-1 div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}#sk-container-id-1 div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}#sk-container-id-1 div.sk-parallel-item:only-child::after {width: 0;}#sk-container-id-1 div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0 0.4em 0.5em 0.4em;box-sizing: border-box;padding-bottom: 0.4em;background-color: white;}#sk-container-id-1 div.sk-label label {font-family: monospace;font-weight: bold;display: inline-block;line-height: 1.2em;}#sk-container-id-1 div.sk-label-container {text-align: center;}#sk-container-id-1 div.sk-container {/* jupyter's `normalize.less` sets `[hidden] { display: none; }` but bootstrap.min.css set `[hidden] { display: none !important; }` so we also need the `!important` here to be able to override the default hidden behavior on the sphinx rendered scikit-learn.org. See: https://github.com/scikit-learn/scikit-learn/issues/21755 */display: inline-block !important;position: relative;}#sk-container-id-1 div.sk-text-repr-fallback {display: none;} Pipeline(steps=[('sfs', SequentialFeatureSelector(estimator=KNeighborsClassifier(n_neighbors=3), k_features=(3, 3), scoring='accuracy')), ('knn2', KNeighborsClassifier(n_neighbors=7))]) In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org. Pipeline Pipeline(steps=[('sfs', SequentialFeatureSelector(estimator=KNeighborsClassifier(n_neighbors=3), k_features=(3, 3), scoring='accuracy')), ('knn2', KNeighborsClassifier(n_neighbors=7))]) sfs: SequentialFeatureSelector SequentialFeatureSelector(estimator=KNeighborsClassifier(n_neighbors=3), k_features=(3, 3), scoring='accuracy') estimator: KNeighborsClassifier KNeighborsClassifier(n_neighbors=3) KNeighborsClassifier KNeighborsClassifier(n_neighbors=3) KNeighborsClassifier KNeighborsClassifier(n_neighbors=7)","title":"Example 8 -- Sequential Feature Selection and GridSearch"},{"location":"user_guide/feature_selection/SequentialFeatureSelector/#example-9-selecting-the-best-feature-combination-in-a-k-range","text":"If k_features is set to to a tuple (min_k, max_k) (new in 0.4.2), the SFS will now select the best feature combination that it discovered by iterating from k=1 to max_k (forward), or max_k to min_k (backward). The size of the returned feature subset is then within max_k to min_k , depending on which combination scored best during cross validation. X.shape (150, 4) from mlxtend.feature_selection import SequentialFeatureSelector as SFS from sklearn.neighbors import KNeighborsClassifier from mlxtend.data import wine_data from sklearn.model_selection import train_test_split from sklearn.preprocessing import StandardScaler from sklearn.pipeline import make_pipeline X, y = wine_data() X_train, X_test, y_train, y_test= train_test_split(X, y, stratify=y, test_size=0.3, random_state=1) knn = KNeighborsClassifier(n_neighbors=2) sfs1 = SFS(estimator=knn, k_features=(3, 10), forward=True, floating=False, scoring='accuracy', cv=5) pipe = make_pipeline(StandardScaler(), sfs1) pipe.fit(X_train, y_train) print('best combination (ACC: %.3f): %s\\n' % (sfs1.k_score_, sfs1.k_feature_idx_)) print('all subsets:\\n', sfs1.subsets_) plot_sfs(sfs1.get_metric_dict(), kind='std_err'); best combination (ACC: 0.992): (0, 1, 2, 3, 6, 8, 9, 10, 11, 12) all subsets: {1: {'feature_idx': (6,), 'cv_scores': array([0.84 , 0.64 , 0.84 , 0.8 , 0.875]), 'avg_score': 0.799, 'feature_names': ('6',)}, 2: {'feature_idx': (6, 9), 'cv_scores': array([0.92 , 0.88 , 1. , 0.96 , 0.91666667]), 'avg_score': 0.9353333333333333, 'feature_names': ('6', '9')}, 3: {'feature_idx': (6, 9, 12), 'cv_scores': array([0.92 , 0.92 , 0.96 , 1. , 0.95833333]), 'avg_score': 0.9516666666666665, 'feature_names': ('6', '9', '12')}, 4: {'feature_idx': (3, 6, 9, 12), 'cv_scores': array([0.96 , 0.96 , 0.96 , 1. , 0.95833333]), 'avg_score': 0.9676666666666666, 'feature_names': ('3', '6', '9', '12')}, 5: {'feature_idx': (3, 6, 9, 10, 12), 'cv_scores': array([0.92, 0.96, 1. , 1. , 1. ]), 'avg_score': 0.976, 'feature_names': ('3', '6', '9', '10', '12')}, 6: {'feature_idx': (2, 3, 6, 9, 10, 12), 'cv_scores': array([0.92, 0.96, 1. , 0.96, 1. ]), 'avg_score': 0.968, 'feature_names': ('2', '3', '6', '9', '10', '12')}, 7: {'feature_idx': (0, 2, 3, 6, 9, 10, 12), 'cv_scores': array([0.92, 0.92, 1. , 1. , 1. ]), 'avg_score': 0.968, 'feature_names': ('0', '2', '3', '6', '9', '10', '12')}, 8: {'feature_idx': (0, 2, 3, 6, 8, 9, 10, 12), 'cv_scores': array([1. , 0.92, 1. , 1. , 1. ]), 'avg_score': 0.984, 'feature_names': ('0', '2', '3', '6', '8', '9', '10', '12')}, 9: {'feature_idx': (0, 2, 3, 6, 8, 9, 10, 11, 12), 'cv_scores': array([1. , 0.92, 1. , 1. , 1. ]), 'avg_score': 0.984, 'feature_names': ('0', '2', '3', '6', '8', '9', '10', '11', '12')}, 10: {'feature_idx': (0, 1, 2, 3, 6, 8, 9, 10, 11, 12), 'cv_scores': array([1. , 0.96, 1. , 1. , 1. ]), 'avg_score': 0.992, 'feature_names': ('0', '1', '2', '3', '6', '8', '9', '10', '11', '12')}}","title":"Example 9 -- Selecting the \"best\"  feature combination in a k-range"},{"location":"user_guide/feature_selection/SequentialFeatureSelector/#example-10-using-other-cross-validation-schemes","text":"In addition to standard k-fold and stratified k-fold, other cross validation schemes can be used with SequentialFeatureSelector . For example, GroupKFold or LeaveOneOut cross-validation from scikit-learn.","title":"Example 10 -- Using other cross-validation schemes"},{"location":"user_guide/feature_selection/SequentialFeatureSelector/#using-groupkfold-with-sequentialfeatureselector","text":"from mlxtend.feature_selection import SequentialFeatureSelector as SFS from sklearn.neighbors import KNeighborsClassifier from mlxtend.data import iris_data from sklearn.model_selection import GroupKFold import numpy as np X, y = iris_data() groups = np.arange(len(y)) // 10 print('groups: {}'.format(groups)) groups: [ 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 2 2 2 2 2 2 2 2 2 2 3 3 3 3 3 3 3 3 3 3 4 4 4 4 4 4 4 4 4 4 5 5 5 5 5 5 5 5 5 5 6 6 6 6 6 6 6 6 6 6 7 7 7 7 7 7 7 7 7 7 8 8 8 8 8 8 8 8 8 8 9 9 9 9 9 9 9 9 9 9 10 10 10 10 10 10 10 10 10 10 11 11 11 11 11 11 11 11 11 11 12 12 12 12 12 12 12 12 12 12 13 13 13 13 13 13 13 13 13 13 14 14 14 14 14 14 14 14 14 14] Calling the split() method of a scikit-learn cross-validator object will return a generator that yields train, test splits. cv_gen = GroupKFold(4).split(X, y, groups) cv_gen <generator object _BaseKFold.split at 0x17c109580> The cv parameter of SequentialFeatureSelector must be either an int or an iterable yielding train, test splits. This iterable can be constructed by passing the train, test split generator to the built-in list() function. cv = list(cv_gen) knn = KNeighborsClassifier(n_neighbors=2) sfs = SFS(estimator=knn, k_features=2, scoring='accuracy', cv=cv) sfs.fit(X, y) print('best combination (ACC: %.3f): %s\\n' % (sfs.k_score_, sfs.k_feature_idx_)) best combination (ACC: 0.940): (2, 3)","title":"Using GroupKFold with SequentialFeatureSelector"},{"location":"user_guide/feature_selection/SequentialFeatureSelector/#example-11-interrupting-long-runs-for-intermediate-results","text":"If your run is taking too long, it is possible to trigger a KeyboardInterrupt (e.g., ctrl+c on a Mac, or interrupting the cell in a Jupyter notebook) to obtain temporary results. Toy dataset from sklearn.datasets import make_classification from sklearn.model_selection import train_test_split X, y = make_classification( n_samples=20000, n_features=500, n_informative=10, n_redundant=40, n_repeated=25, n_clusters_per_class=5, flip_y=0.05, class_sep=0.5, random_state=123, ) X_train, X_test, y_train, y_test = train_test_split( X, y, test_size=0.2, random_state=123 ) Long run with interruption from mlxtend.feature_selection import SequentialFeatureSelector as SFS from sklearn.linear_model import LogisticRegression model = LogisticRegression() sfs1 = SFS(model, k_features=10, forward=True, floating=False, verbose=2, scoring='accuracy', cv=5) sfs1 = sfs1.fit(X_train, y_train) [Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers. [Parallel(n_jobs=1)]: Done 1 out of 1 | elapsed: 0.0s remaining: 0.0s [Parallel(n_jobs=1)]: Done 500 out of 500 | elapsed: 8.3s finished [2023-05-17 08:36:32] Features: 1/10 -- score: 0.5965[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers. [Parallel(n_jobs=1)]: Done 1 out of 1 | elapsed: 0.0s remaining: 0.0s [Parallel(n_jobs=1)]: Done 499 out of 499 | elapsed: 13.8s finished [2023-05-17 08:36:45] Features: 2/10 -- score: 0.6256875000000001[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers. [Parallel(n_jobs=1)]: Done 1 out of 1 | elapsed: 0.0s remaining: 0.0s [Parallel(n_jobs=1)]: Done 498 out of 498 | elapsed: 18.1s finished [2023-05-17 08:37:03] Features: 3/10 -- score: 0.642[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers. [Parallel(n_jobs=1)]: Done 1 out of 1 | elapsed: 0.0s remaining: 0.0s [Parallel(n_jobs=1)]: Done 497 out of 497 | elapsed: 20.4s finished [2023-05-17 08:37:24] Features: 4/10 -- score: 0.6463125[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers. [Parallel(n_jobs=1)]: Done 1 out of 1 | elapsed: 0.0s remaining: 0.0s [Parallel(n_jobs=1)]: Done 496 out of 496 | elapsed: 22.2s finished [2023-05-17 08:37:46] Features: 5/10 -- score: 0.6495000000000001[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers. [Parallel(n_jobs=1)]: Done 1 out of 1 | elapsed: 0.1s remaining: 0.0s [Parallel(n_jobs=1)]: Done 495 out of 495 | elapsed: 26.1s finished [2023-05-17 08:38:12] Features: 6/10 -- score: 0.6514374999999999[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers. [Parallel(n_jobs=1)]: Done 1 out of 1 | elapsed: 0.0s remaining: 0.0s [Parallel(n_jobs=1)]: Done 494 out of 494 | elapsed: 26.1s finished [2023-05-17 08:38:38] Features: 7/10 -- score: 0.6533749999999999[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers. [Parallel(n_jobs=1)]: Done 1 out of 1 | elapsed: 0.0s remaining: 0.0s [Parallel(n_jobs=1)]: Done 493 out of 493 | elapsed: 25.3s finished [2023-05-17 08:39:04] Features: 8/10 -- score: 0.6545624999999999[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers. [Parallel(n_jobs=1)]: Done 1 out of 1 | elapsed: 0.1s remaining: 0.0s [Parallel(n_jobs=1)]: Done 492 out of 492 | elapsed: 26.3s finished [2023-05-17 08:39:30] Features: 9/10 -- score: 0.6549375[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers. [Parallel(n_jobs=1)]: Done 1 out of 1 | elapsed: 0.1s remaining: 0.0s [Parallel(n_jobs=1)]: Done 491 out of 491 | elapsed: 27.0s finished [2023-05-17 08:39:57] Features: 10/10 -- score: 0.6554374999999999 Finalizing the fit Note that the feature selection run hasn't finished, so certain attributes may not be available. In order to use the SFS instance, it is recommended to call finalize_fit , which will make SFS estimator appear as \"fitted\" process the temporary results: sfs1.finalize_fit() print(sfs1.k_feature_idx_) print(sfs1.k_score_) (30, 128, 144, 160, 184, 229, 256, 356, 439, 458) 0.6554374999999999","title":"Example 11 - Interrupting Long Runs for Intermediate Results"},{"location":"user_guide/feature_selection/SequentialFeatureSelector/#example-12-using-pandas-dataframes","text":"Optionally, we can also use pandas DataFrames and pandas Series as input to the fit function. In this case, the column names of the pandas DataFrame will be used as feature names. However, note that if custom_feature_names are provided in the fit function, these custom_feature_names take precedence over the DataFrame column-based feature names. import pandas as pd from sklearn.neighbors import KNeighborsClassifier from sklearn.datasets import load_iris from mlxtend.feature_selection import SequentialFeatureSelector as SFS iris = load_iris() X = iris.data y = iris.target knn = KNeighborsClassifier(n_neighbors=4) sfs1 = SFS(knn, k_features=3, forward=True, floating=False, scoring='accuracy', cv=0) X_df = pd.DataFrame(X, columns=['sepal len', 'petal len', 'sepal width', 'petal width']) X_df.head() .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } sepal len petal len sepal width petal width 0 5.1 3.5 1.4 0.2 1 4.9 3.0 1.4 0.2 2 4.7 3.2 1.3 0.2 3 4.6 3.1 1.5 0.2 4 5.0 3.6 1.4 0.2 Also, the target array, y , can be optionally be cast as a Series: y_series = pd.Series(y) y_series.head() 0 0 1 0 2 0 3 0 4 0 dtype: int64 sfs1 = sfs1.fit(X_df, y_series) Note that the only difference of passing a pandas DataFrame as input is that the sfs1.subsets_ array will now contain a new column, sfs1.subsets_ {1: {'feature_idx': (3,), 'cv_scores': array([0.96]), 'avg_score': 0.96, 'feature_names': ('petal width',)}, 2: {'feature_idx': (2, 3), 'cv_scores': array([0.97333333]), 'avg_score': 0.9733333333333334, 'feature_names': ('sepal width', 'petal width')}, 3: {'feature_idx': (1, 2, 3), 'cv_scores': array([0.97333333]), 'avg_score': 0.9733333333333334, 'feature_names': ('petal len', 'sepal width', 'petal width')}} In mlxtend version >= 0.13 pandas DataFrames are supported as feature inputs to the SequentianFeatureSelector instead of NumPy arrays or other NumPy-like array types.","title":"Example 12 - Using Pandas DataFrames"},{"location":"user_guide/feature_selection/SequentialFeatureSelector/#example-13-specifying-fixed-feature-sets","text":"Often, it may be useful to specify a fixed set of features we want to use for a given model (e.g., determined by prior knowledge or domain knowledge). Since MLxtend v 0.18.0, it is now possible to specify such features via the fixed_features attribute. This will mean that these features are guaranteed to be included in the selected subsets. Note that this feature works for all options regarding forward and backward selection, and using floating selection or not. The example below illustrates how we can set the features 0 and 2 in the dataset as fixed: from sklearn.neighbors import KNeighborsClassifier from sklearn.datasets import load_iris iris = load_iris() X = iris.data y = iris.target knn = KNeighborsClassifier(n_neighbors=3) from mlxtend.feature_selection import SequentialFeatureSelector as SFS sfs1 = SFS(knn, k_features=4, forward=True, floating=False, verbose=2, scoring='accuracy', fixed_features=(0, 2), cv=3) sfs1 = sfs1.fit(X, y) [Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers. [Parallel(n_jobs=1)]: Done 1 out of 1 | elapsed: 0.0s remaining: 0.0s [Parallel(n_jobs=1)]: Done 2 out of 2 | elapsed: 0.0s finished [2023-05-17 08:39:57] Features: 3/4 -- score: 0.9733333333333333[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers. [Parallel(n_jobs=1)]: Done 1 out of 1 | elapsed: 0.0s remaining: 0.0s [Parallel(n_jobs=1)]: Done 1 out of 1 | elapsed: 0.0s finished [2023-05-17 08:39:57] Features: 4/4 -- score: 0.9733333333333333 sfs1.subsets_ {2: {'feature_idx': (0, 2), 'cv_scores': array([0.98, 0.92, 0.94]), 'avg_score': 0.9466666666666667, 'feature_names': ('0', '2')}, 3: {'feature_idx': (0, 2, 3), 'cv_scores': array([0.98, 0.96, 0.98]), 'avg_score': 0.9733333333333333, 'feature_names': ('0', '2', '3')}, 4: {'feature_idx': (0, 1, 2, 3), 'cv_scores': array([0.98, 0.96, 0.98]), 'avg_score': 0.9733333333333333, 'feature_names': ('0', '1', '2', '3')}} If the input dataset is a pandas DataFrame, we can also use the column names directly: import pandas as pd X_df = pd.DataFrame(X, columns=['sepal len', 'petal len', 'sepal width', 'petal width']) X_df.head() .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } sepal len petal len sepal width petal width 0 5.1 3.5 1.4 0.2 1 4.9 3.0 1.4 0.2 2 4.7 3.2 1.3 0.2 3 4.6 3.1 1.5 0.2 4 5.0 3.6 1.4 0.2 sfs2 = SFS(knn, k_features=4, forward=True, floating=False, verbose=2, scoring='accuracy', fixed_features=('sepal len', 'petal len'), cv=3) sfs2 = sfs2.fit(X_df, y_series) [Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers. [Parallel(n_jobs=1)]: Done 1 out of 1 | elapsed: 0.0s remaining: 0.0s [Parallel(n_jobs=1)]: Done 2 out of 2 | elapsed: 0.0s finished [2023-05-17 08:39:57] Features: 3/4 -- score: 0.9466666666666667[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers. [Parallel(n_jobs=1)]: Done 1 out of 1 | elapsed: 0.0s remaining: 0.0s [Parallel(n_jobs=1)]: Done 1 out of 1 | elapsed: 0.0s finished [2023-05-17 08:39:57] Features: 4/4 -- score: 0.9733333333333333 sfs2.subsets_ {2: {'feature_idx': (0, 1), 'cv_scores': array([0.72, 0.74, 0.78]), 'avg_score': 0.7466666666666667, 'feature_names': ('sepal len', 'petal len')}, 3: {'feature_idx': (0, 1, 2), 'cv_scores': array([0.98, 0.92, 0.94]), 'avg_score': 0.9466666666666667, 'feature_names': ('sepal len', 'petal len', 'sepal width')}, 4: {'feature_idx': (0, 1, 2, 3), 'cv_scores': array([0.98, 0.96, 0.98]), 'avg_score': 0.9733333333333333, 'feature_names': ('sepal len', 'petal len', 'sepal width', 'petal width')}}","title":"Example 13 - Specifying Fixed Feature Sets"},{"location":"user_guide/feature_selection/SequentialFeatureSelector/#example-13-working-with-feature-groups","text":"Since mlxtend v0.21.0, it is possible to specify feature groups. Feature groups allow you to group certain features together, such that they are always selected as a group. This can be very useful in contexts similar to one-hot encoding -- if you want to treat the one-hot encoded feature as a single feature: In the following example, we specify sepal length and sepal width as a feature group so that they are always selected together: from sklearn.datasets import load_iris import pandas as pd iris = load_iris() X = iris.data y = iris.target X_df = pd.DataFrame(X, columns=['sepal len', 'petal len', 'sepal wid', 'petal wid']) X_df.head() .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } sepal len petal len sepal wid petal wid 0 5.1 3.5 1.4 0.2 1 4.9 3.0 1.4 0.2 2 4.7 3.2 1.3 0.2 3 4.6 3.1 1.5 0.2 4 5.0 3.6 1.4 0.2 from sklearn.neighbors import KNeighborsClassifier from mlxtend.feature_selection import SequentialFeatureSelector as SFS knn = KNeighborsClassifier(n_neighbors=3) sfs1 = SFS(knn, k_features=2, scoring='accuracy', feature_groups=(['sepal len', 'sepal wid'], ['petal len'], ['petal wid']), cv=3) sfs1 = sfs1.fit(X_df, y) sfs1 = SFS(knn, k_features=2, scoring='accuracy', feature_groups=[[0, 2], [1], [3]], cv=3) sfs1 = sfs1.fit(X, y)","title":"Example 13 - Working with Feature Groups"},{"location":"user_guide/feature_selection/SequentialFeatureSelector/#example-14-multiclass-metrics","text":"Certain scoring metrics like ROC AUC are originally designed for binary classification. However, they can also be used for multiclass settings. It is best to consult this scikit-learn metrics table for this. For example, we can use a ROC AUC One-Vs-Rest score via \u2018\"roc_auc_ovr\" as shown below. from sklearn.datasets import make_blobs X, y = make_blobs(n_samples=10, centers=4, n_features=5, random_state=0) from mlxtend.feature_selection import SequentialFeatureSelector as SFS sfs1 = SFS(knn, k_features=3, forward=True, floating=False, verbose=2, scoring='roc_auc_ovr', cv=0) sfs1 = sfs1.fit(X, y) [Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers. [Parallel(n_jobs=1)]: Done 1 out of 1 | elapsed: 0.0s remaining: 0.0s [Parallel(n_jobs=1)]: Done 5 out of 5 | elapsed: 0.0s finished [2023-05-17 08:39:57] Features: 1/3 -- score: 1.0[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers. [Parallel(n_jobs=1)]: Done 1 out of 1 | elapsed: 0.0s remaining: 0.0s [Parallel(n_jobs=1)]: Done 4 out of 4 | elapsed: 0.0s finished [2023-05-17 08:39:57] Features: 2/3 -- score: 1.0[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers. [Parallel(n_jobs=1)]: Done 1 out of 1 | elapsed: 0.0s remaining: 0.0s [Parallel(n_jobs=1)]: Done 3 out of 3 | elapsed: 0.0s finished [2023-05-17 08:39:57] Features: 3/3 -- score: 1.0","title":"Example 14 - Multiclass Metrics"},{"location":"user_guide/feature_selection/SequentialFeatureSelector/#api","text":"SequentialFeatureSelector(estimator, k_features=1, forward=True, floating=False, verbose=0, scoring=None, cv=5, n_jobs=1, pre_dispatch='2 n_jobs', clone_estimator=True, fixed_features=None, feature_groups=None)* Sequential Feature Selection for Classification and Regression. Parameters estimator : scikit-learn classifier or regressor k_features : int or tuple or str (default: 1) Number of features to select, where k_features < the full feature set. New in 0.4.2: A tuple containing a min and max value can be provided, and the SFS will consider return any feature combination between min and max that scored highest in cross-validation. For example, the tuple (1, 4) will return any combination from 1 up to 4 features instead of a fixed number of features k. New in 0.8.0: A string argument \"best\" or \"parsimonious\". If \"best\" is provided, the feature selector will return the feature subset with the best cross-validation performance. If \"parsimonious\" is provided as an argument, the smallest feature subset that is within one standard error of the cross-validation performance will be selected. forward : bool (default: True) Forward selection if True, backward selection otherwise floating : bool (default: False) Adds a conditional exclusion/inclusion if True. verbose : int (default: 0), level of verbosity to use in logging. If 0, no output, if 1 number of features in current set, if 2 detailed logging i ncluding timestamp and cv scores at step. scoring : str, callable, or None (default: None) If None (default), uses 'accuracy' for sklearn classifiers and 'r2' for sklearn regressors. If str, uses a sklearn scoring metric string identifier, for example {accuracy, f1, precision, recall, roc_auc} for classifiers, {'mean_absolute_error', 'mean_squared_error'/'neg_mean_squared_error', 'median_absolute_error', 'r2'} for regressors. If a callable object or function is provided, it has to be conform with sklearn's signature scorer(estimator, X, y) ; see https://scikit-learn.org/stable/modules/generated/sklearn.metrics.make_scorer.html for more information. cv : int (default: 5) Integer or iterable yielding train, test splits. If cv is an integer and estimator is a classifier (or y consists of integer class labels) stratified k-fold. Otherwise regular k-fold cross-validation is performed. No cross-validation if cv is None, False, or 0. n_jobs : int (default: 1) The number of CPUs to use for evaluating different feature subsets in parallel. -1 means 'all CPUs'. pre_dispatch : int, or string (default: '2*n_jobs') Controls the number of jobs that get dispatched during parallel execution if n_jobs > 1 or n_jobs=-1 . Reducing this number can be useful to avoid an explosion of memory consumption when more jobs get dispatched than CPUs can process. This parameter can be: None, in which case all the jobs are immediately created and spawned. Use this for lightweight and fast-running jobs, to avoid delays due to on-demand spawning of the jobs An int, giving the exact number of total jobs that are spawned A string, giving an expression as a function of n_jobs, as in 2*n_jobs clone_estimator : bool (default: True) Clones estimator if True; works with the original estimator instance if False. Set to False if the estimator doesn't implement scikit-learn's set_params and get_params methods. In addition, it is required to set cv=0, and n_jobs=1. fixed_features : tuple (default: None) If not None , the feature indices provided as a tuple will be regarded as fixed by the feature selector. For example, if fixed_features=(1, 3, 7) , the 2nd, 4th, and 8th feature are guaranteed to be present in the solution. Note that if fixed_features is not None , make sure that the number of features to be selected is greater than len(fixed_features) . In other words, ensure that k_features > len(fixed_features) . New in mlxtend v. 0.18.0. feature_groups : list or None (default: None) Optional argument for treating certain features as a group. This means, the features within a group are always selected together, never split. For example, feature_groups=[[1], [2], [3, 4, 5]] specifies 3 feature groups. In this case, possible feature selection results with k_features=2 are [[1], [2] , [[1], [3, 4, 5]] , or [[2], [3, 4, 5]] . Feature groups can be useful for interpretability, for example, if features 3, 4, 5 are one-hot encoded features. (For more details, please read the notes at the bottom of this docstring). New in mlxtend v. 0.21.0. Attributes k_feature_idx_ : array-like, shape = [n_predictions] Feature Indices of the selected feature subsets. k_feature_names_ : array-like, shape = [n_predictions] Feature names of the selected feature subsets. If pandas DataFrames are used in the fit method, the feature names correspond to the column names. Otherwise, the feature names are string representation of the feature array indices. New in v 0.13.0. k_score_ : float Cross validation average score of the selected subset. subsets_ : dict A dictionary of selected feature subsets during the sequential selection, where the dictionary keys are the lengths k of these feature subsets. If the parameter feature_groups is not None, the value of key indicates the number of groups that are selected together. The dictionary values are dictionaries themselves with the following keys: 'feature_idx' (tuple of indices of the feature subset) 'feature_names' (tuple of feature names of the feat. subset) 'cv_scores' (list individual cross-validation scores) 'avg_score' (average cross-validation score) Note that if pandas DataFrames are used in the fit method, the 'feature_names' correspond to the column names. Otherwise, the feature names are string representation of the feature array indices. The 'feature_names' is new in v 0.13.0. Notes (1) If parameter feature_groups is not None, the number of features is equal to the number of feature groups, i.e. len(feature_groups) . For example, if feature_groups = [[0], [1], [2, 3], [4]] , then the max_features value cannot exceed 4. (2) Although two or more individual features may be considered as one group throughout the feature-selection process, it does not mean the individual features of that group have the same impact on the outcome. For instance, in linear regression, the coefficient of the feature 2 and 3 can be different even if they are considered as one group in feature_groups. (3) If both fixed_features and feature_groups are specified, ensure that each feature group contains the fixed_features selection. E.g., for a 3-feature set fixed_features=[0, 1] and feature_groups=[[0, 1], [2]] is valid; fixed_features=[0, 1] and feature_groups=[[0], [1, 2]] is not valid. (4) In case of KeyboardInterrupt, the dictionary subsets may not be completed. If user is still interested in getting the best score, they can use method `finalize_fit`. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/feature_selection/SequentialFeatureSelector/","title":"API"},{"location":"user_guide/feature_selection/SequentialFeatureSelector/#methods","text":"finalize_fit() None fit(X, y, groups=None, fit_params) Perform feature selection and learn model from training data. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. New in v 0.13.0: pandas DataFrames are now also accepted as argument for X. y : array-like, shape = [n_samples] Target values. New in v 0.13.0: pandas DataFrames are now also accepted as argument for y. groups : array-like, with shape (n_samples,), optional Group labels for the samples used while splitting the dataset into train/test set. Passed to the fit method of the cross-validator. fit_params : various, optional Additional parameters that are being passed to the estimator. For example, sample_weights=weights . Returns self : object fit_transform(X, y, groups=None, fit_params) Fit to training data then reduce X to its most important features. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. New in v 0.13.0: pandas DataFrames are now also accepted as argument for X. y : array-like, shape = [n_samples] Target values. New in v 0.13.0: a pandas Series are now also accepted as argument for y. groups : array-like, with shape (n_samples,), optional Group labels for the samples used while splitting the dataset into train/test set. Passed to the fit method of the cross-validator. fit_params : various, optional Additional parameters that are being passed to the estimator. For example, sample_weights=weights . Returns Reduced feature subset of X, shape={n_samples, k_features} generate_error_message_k_features(name) None get_metric_dict(confidence_interval=0.95) Return metric dictionary Parameters confidence_interval : float (default: 0.95) A positive float between 0.0 and 1.0 to compute the confidence interval bounds of the CV score averages. Returns Dictionary with items where each dictionary value is a list with the number of iterations (number of feature subsets) as its length. The dictionary keys corresponding to these lists are as follows: 'feature_idx': tuple of the indices of the feature subset 'cv_scores': list with individual CV scores 'avg_score': of CV average scores 'std_dev': standard deviation of the CV score average 'std_err': standard error of the CV score average 'ci_bound': confidence interval bound of the CV score average get_params(deep=True) Get parameters for this estimator. Parameters deep : bool, default=True If True, will return the parameters for this estimator and contained subobjects that are estimators. Returns params : dict Parameter names mapped to their values. set_params( params) Set the parameters of this estimator. Valid parameter keys can be listed with get_params() . Returns self transform(X) Reduce X to its most important features. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. New in v 0.13.0: pandas DataFrames are now also accepted as argument for X. Returns Reduced feature subset of X, shape={n_samples, k_features}","title":"Methods"},{"location":"user_guide/feature_selection/SequentialFeatureSelector/#properties","text":"named_estimators Returns List of named estimator tuples, like [('svc', SVC(...))] ython","title":"Properties"},{"location":"user_guide/file_io/find_filegroups/","text":"find_filegroups: Find files that only differ via their file extensions A function that finds files that belong together (i.e., differ only by file extension) in different directories and collects them in a Python dictionary for further processing tasks. from mlxtend.file_io import find_filegroups Overview This function finds files that are related to each other based on their file names. This can be useful for parsing collections files that have been stored in different subdirectories, for examples: input_dir/ task01.txt task02.txt ... log_dir/ task01.log task02.log ... output_dir/ task01.dat task02.dat ... References - Example 1 - Grouping related files in a dictionary Given the following directory and file structure dir_1/ file_1.log file_2.log file_3.log dir_2/ file_1.csv file_2.csv file_3.csv dir_3/ file_1.txt file_2.txt file_3.txt we can use find_filegroups to group related files as items of a dictionary as shown below: from mlxtend.file_io import find_filegroups find_filegroups(paths=['./data_find_filegroups/dir_1', './data_find_filegroups/dir_2', './data_find_filegroups/dir_3'], substring='file_') {'file_1': ['./data_find_filegroups/dir_1/file_1.log', './data_find_filegroups/dir_2/file_1.csv', './data_find_filegroups/dir_3/file_1.txt'], 'file_2': ['./data_find_filegroups/dir_1/file_2.log', './data_find_filegroups/dir_2/file_2.csv', './data_find_filegroups/dir_3/file_2.txt'], 'file_3': ['./data_find_filegroups/dir_1/file_3.log', './data_find_filegroups/dir_2/file_3.csv', './data_find_filegroups/dir_3/file_3.txt']} API find_filegroups(paths, substring='', extensions=None, validity_check=True, ignore_invisible=True, rstrip='', ignore_substring=None) Find and collect files from different directories in a python dictionary. Parameters paths : list Paths of the directories to be searched. Dictionary keys are build from the first directory. substring : str (default: '') Substring that all files have to contain to be considered. extensions : list (default: None) None or list of allowed file extensions for each path. If provided, the number of extensions must match the number of paths . validity_check : bool (default: None) If True , checks if all dictionary values have the same number of file paths. Prints a warning and returns an empty dictionary if the validity check failed. ignore_invisible : bool (default: True) If True , ignores invisible files (i.e., files starting with a period). rstrip : str (default: '') If provided, strips characters from right side of the file base names after splitting the extension. Useful to trim different filenames to a common stem. E.g,. \"abc_d.txt\" and \"abc_d_.csv\" would share the stem \"abc_d\" if rstrip is set to \"_\". ignore_substring : str (default: None) Ignores files that contain the specified substring. Returns groups : dict Dictionary of files paths. Keys are the file names found in the first directory listed in paths (without file extension). Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/file_io/find_filegroups/","title":"find_filegroups: Find files that only differ via their file extensions"},{"location":"user_guide/file_io/find_filegroups/#find_filegroups-find-files-that-only-differ-via-their-file-extensions","text":"A function that finds files that belong together (i.e., differ only by file extension) in different directories and collects them in a Python dictionary for further processing tasks. from mlxtend.file_io import find_filegroups","title":"find_filegroups: Find files that only differ via their file extensions"},{"location":"user_guide/file_io/find_filegroups/#overview","text":"This function finds files that are related to each other based on their file names. This can be useful for parsing collections files that have been stored in different subdirectories, for examples: input_dir/ task01.txt task02.txt ... log_dir/ task01.log task02.log ... output_dir/ task01.dat task02.dat ...","title":"Overview"},{"location":"user_guide/file_io/find_filegroups/#references","text":"-","title":"References"},{"location":"user_guide/file_io/find_filegroups/#example-1-grouping-related-files-in-a-dictionary","text":"Given the following directory and file structure dir_1/ file_1.log file_2.log file_3.log dir_2/ file_1.csv file_2.csv file_3.csv dir_3/ file_1.txt file_2.txt file_3.txt we can use find_filegroups to group related files as items of a dictionary as shown below: from mlxtend.file_io import find_filegroups find_filegroups(paths=['./data_find_filegroups/dir_1', './data_find_filegroups/dir_2', './data_find_filegroups/dir_3'], substring='file_') {'file_1': ['./data_find_filegroups/dir_1/file_1.log', './data_find_filegroups/dir_2/file_1.csv', './data_find_filegroups/dir_3/file_1.txt'], 'file_2': ['./data_find_filegroups/dir_1/file_2.log', './data_find_filegroups/dir_2/file_2.csv', './data_find_filegroups/dir_3/file_2.txt'], 'file_3': ['./data_find_filegroups/dir_1/file_3.log', './data_find_filegroups/dir_2/file_3.csv', './data_find_filegroups/dir_3/file_3.txt']}","title":"Example 1 - Grouping related files in a dictionary"},{"location":"user_guide/file_io/find_filegroups/#api","text":"find_filegroups(paths, substring='', extensions=None, validity_check=True, ignore_invisible=True, rstrip='', ignore_substring=None) Find and collect files from different directories in a python dictionary. Parameters paths : list Paths of the directories to be searched. Dictionary keys are build from the first directory. substring : str (default: '') Substring that all files have to contain to be considered. extensions : list (default: None) None or list of allowed file extensions for each path. If provided, the number of extensions must match the number of paths . validity_check : bool (default: None) If True , checks if all dictionary values have the same number of file paths. Prints a warning and returns an empty dictionary if the validity check failed. ignore_invisible : bool (default: True) If True , ignores invisible files (i.e., files starting with a period). rstrip : str (default: '') If provided, strips characters from right side of the file base names after splitting the extension. Useful to trim different filenames to a common stem. E.g,. \"abc_d.txt\" and \"abc_d_.csv\" would share the stem \"abc_d\" if rstrip is set to \"_\". ignore_substring : str (default: None) Ignores files that contain the specified substring. Returns groups : dict Dictionary of files paths. Keys are the file names found in the first directory listed in paths (without file extension). Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/file_io/find_filegroups/","title":"API"},{"location":"user_guide/file_io/find_files/","text":"find_files: Find files based on substring matches A function that finds files in a given directory based on substring matches and returns a list of the file names found. from mlxtend.file_io import find_files Overview This function finds files based on substring search. This is especially useful if we want to find specific files in a directory tree and return their absolute paths for further processing in Python. References - Example 1 - Grouping related files in a dictionary Given the following directory and file structure dir_1/ file_1.log file_2.log file_3.log dir_2/ file_1.csv file_2.csv file_3.csv dir_3/ file_1.txt file_2.txt file_3.txt we can use find_files to return the paths to all files that contain the substring _2 as follows: from mlxtend.file_io import find_files find_files(substring='_2', path='./data_find_filegroups/', recursive=True) ['./data_find_filegroups/dir_1/file_2.log', './data_find_filegroups/dir_2/file_2.csv', './data_find_filegroups/dir_3/file_2.txt'] API find_files(substring, path, recursive=False, check_ext=None, ignore_invisible=True, ignore_substring=None) Find files in a directory based on substring matching. Parameters substring : str Substring of the file to be matched. path : str Path where to look. recursive : bool If true, searches subdirectories recursively. check_ext : str If string (e.g., '.txt'), only returns files that match the specified file extension. ignore_invisible : bool If True , ignores invisible files (i.e., files starting with a period). ignore_substring : str Ignores files that contain the specified substring. Returns results : list List of the matched files. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/file_io/find_files/","title":"find_files: Find files based on substring matches"},{"location":"user_guide/file_io/find_files/#find_files-find-files-based-on-substring-matches","text":"A function that finds files in a given directory based on substring matches and returns a list of the file names found. from mlxtend.file_io import find_files","title":"find_files: Find files based on substring matches"},{"location":"user_guide/file_io/find_files/#overview","text":"This function finds files based on substring search. This is especially useful if we want to find specific files in a directory tree and return their absolute paths for further processing in Python.","title":"Overview"},{"location":"user_guide/file_io/find_files/#references","text":"-","title":"References"},{"location":"user_guide/file_io/find_files/#example-1-grouping-related-files-in-a-dictionary","text":"Given the following directory and file structure dir_1/ file_1.log file_2.log file_3.log dir_2/ file_1.csv file_2.csv file_3.csv dir_3/ file_1.txt file_2.txt file_3.txt we can use find_files to return the paths to all files that contain the substring _2 as follows: from mlxtend.file_io import find_files find_files(substring='_2', path='./data_find_filegroups/', recursive=True) ['./data_find_filegroups/dir_1/file_2.log', './data_find_filegroups/dir_2/file_2.csv', './data_find_filegroups/dir_3/file_2.txt']","title":"Example 1 - Grouping related files in a dictionary"},{"location":"user_guide/file_io/find_files/#api","text":"find_files(substring, path, recursive=False, check_ext=None, ignore_invisible=True, ignore_substring=None) Find files in a directory based on substring matching. Parameters substring : str Substring of the file to be matched. path : str Path where to look. recursive : bool If true, searches subdirectories recursively. check_ext : str If string (e.g., '.txt'), only returns files that match the specified file extension. ignore_invisible : bool If True , ignores invisible files (i.e., files starting with a period). ignore_substring : str Ignores files that contain the specified substring. Returns results : list List of the matched files. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/file_io/find_files/","title":"API"},{"location":"user_guide/frequent_patterns/apriori/","text":"apriori: Frequent itemsets via the Apriori algorithm Apriori function to extract frequent itemsets for association rule mining from mlxtend.frequent_patterns import apriori Overview Apriori is a popular algorithm [1] for extracting frequent itemsets with applications in association rule learning. The apriori algorithm has been designed to operate on databases containing transactions, such as purchases by customers of a store. An itemset is considered as \"frequent\" if it meets a user-specified support threshold. For instance, if the support threshold is set to 0.5 (50%), a frequent itemset is defined as a set of items that occur together in at least 50% of all transactions in the database. References [1] Agrawal, Rakesh, and Ramakrishnan Srikant. \" Fast algorithms for mining association rules .\" Proc. 20th int. conf. very large data bases, VLDB. Vol. 1215. 1994. Related FP-Growth FP-Max Example 1 -- Generating Frequent Itemsets The apriori function expects data in a one-hot encoded pandas DataFrame. Suppose we have the following transaction data: dataset = [['Milk', 'Onion', 'Nutmeg', 'Kidney Beans', 'Eggs', 'Yogurt'], ['Dill', 'Onion', 'Nutmeg', 'Kidney Beans', 'Eggs', 'Yogurt'], ['Milk', 'Apple', 'Kidney Beans', 'Eggs'], ['Milk', 'Unicorn', 'Corn', 'Kidney Beans', 'Yogurt'], ['Corn', 'Onion', 'Onion', 'Kidney Beans', 'Ice cream', 'Eggs']] We can transform it into the right format via the TransactionEncoder as follows: import pandas as pd from mlxtend.preprocessing import TransactionEncoder te = TransactionEncoder() te_ary = te.fit(dataset).transform(dataset) df = pd.DataFrame(te_ary, columns=te.columns_) df .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } Apple Corn Dill Eggs Ice cream Kidney Beans Milk Nutmeg Onion Unicorn Yogurt 0 False False False True False True True True True False True 1 False False True True False True False True True False True 2 True False False True False True True False False False False 3 False True False False False True True False False True True 4 False True False True True True False False True False False Now, let us return the items and itemsets with at least 60% support: from mlxtend.frequent_patterns import apriori apriori(df, min_support=0.6) .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } support itemsets 0 0.8 (3) 1 1.0 (5) 2 0.6 (6) 3 0.6 (8) 4 0.6 (10) 5 0.8 (3, 5) 6 0.6 (8, 3) 7 0.6 (5, 6) 8 0.6 (8, 5) 9 0.6 (10, 5) 10 0.6 (8, 3, 5) By default, apriori returns the column indices of the items, which may be useful in downstream operations such as association rule mining. For better readability, we can set use_colnames=True to convert these integer values into the respective item names: apriori(df, min_support=0.6, use_colnames=True) .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } support itemsets 0 0.8 (Eggs) 1 1.0 (Kidney Beans) 2 0.6 (Milk) 3 0.6 (Onion) 4 0.6 (Yogurt) 5 0.8 (Eggs, Kidney Beans) 6 0.6 (Eggs, Onion) 7 0.6 (Kidney Beans, Milk) 8 0.6 (Kidney Beans, Onion) 9 0.6 (Yogurt, Kidney Beans) 10 0.6 (Kidney Beans, Eggs, Onion) Example 2 -- Selecting and Filtering Results The advantage of working with pandas DataFrames is that we can use its convenient features to filter the results. For instance, let's assume we are only interested in itemsets of length 2 that have a support of at least 80 percent. First, we create the frequent itemsets via apriori and add a new column that stores the length of each itemset: frequent_itemsets = apriori(df, min_support=0.6, use_colnames=True) frequent_itemsets['length'] = frequent_itemsets['itemsets'].apply(lambda x: len(x)) frequent_itemsets .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } support itemsets length 0 0.8 (Eggs) 1 1 1.0 (Kidney Beans) 1 2 0.6 (Milk) 1 3 0.6 (Onion) 1 4 0.6 (Yogurt) 1 5 0.8 (Eggs, Kidney Beans) 2 6 0.6 (Eggs, Onion) 2 7 0.6 (Kidney Beans, Milk) 2 8 0.6 (Kidney Beans, Onion) 2 9 0.6 (Yogurt, Kidney Beans) 2 10 0.6 (Kidney Beans, Eggs, Onion) 3 Then, we can select the results that satisfy our desired criteria as follows: frequent_itemsets[ (frequent_itemsets['length'] == 2) & (frequent_itemsets['support'] >= 0.8) ] .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } support itemsets length 5 0.8 (Eggs, Kidney Beans) 2 Similarly, using the Pandas API, we can select entries based on the \"itemsets\" column: frequent_itemsets[ frequent_itemsets['itemsets'] == {'Onion', 'Eggs'} ] .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } support itemsets length 6 0.6 (Eggs, Onion) 2 Frozensets Note that the entries in the \"itemsets\" column are of type frozenset , which is built-in Python type that is similar to a Python set but immutable, which makes it more efficient for certain query or comparison operations (https://docs.python.org/3.6/library/stdtypes.html#frozenset). Since frozenset s are sets, the item order does not matter. I.e., the query frequent_itemsets[ frequent_itemsets['itemsets'] == {'Onion', 'Eggs'} ] is equivalent to any of the following three frequent_itemsets[ frequent_itemsets['itemsets'] == {'Eggs', 'Onion'} ] frequent_itemsets[ frequent_itemsets['itemsets'] == frozenset(('Eggs', 'Onion')) ] frequent_itemsets[ frequent_itemsets['itemsets'] == frozenset(('Onion', 'Eggs')) ] Example 3 -- Working with Sparse Representations To save memory, you may want to represent your transaction data in the sparse format. This is especially useful if you have lots of products and small transactions. oht_ary = te.fit(dataset).transform(dataset, sparse=True) sparse_df = pd.DataFrame.sparse.from_spmatrix(oht_ary, columns=te.columns_) sparse_df .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } Apple Corn Dill Eggs Ice cream Kidney Beans Milk Nutmeg Onion Unicorn Yogurt 0 False False False True False True True True True False True 1 False False True True False True False True True False True 2 True False False True False True True False False False False 3 False True False False False True True False False True True 4 False True False True True True False False True False False apriori(sparse_df, min_support=0.6, use_colnames=True, verbose=1) Processing 21 combinations | Sampling itemset size 3 .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } support itemsets 0 0.8 (Eggs) 1 1.0 (Kidney Beans) 2 0.6 (Milk) 3 0.6 (Onion) 4 0.6 (Yogurt) 5 0.8 (Eggs, Kidney Beans) 6 0.6 (Eggs, Onion) 7 0.6 (Kidney Beans, Milk) 8 0.6 (Kidney Beans, Onion) 9 0.6 (Yogurt, Kidney Beans) 10 0.6 (Kidney Beans, Eggs, Onion) API apriori(df, min_support=0.5, use_colnames=False, max_len=None, verbose=0, low_memory=False) Get frequent itemsets from a one-hot DataFrame Parameters df : pandas DataFrame pandas DataFrame the encoded format. Also supports DataFrames with sparse data; for more info, please see (https://pandas.pydata.org/pandas-docs/stable/ user_guide/sparse.html#sparse-data-structures) Please note that the old pandas SparseDataFrame format is no longer supported in mlxtend >= 0.17.2. The allowed values are either 0/1 or True/False. For example, Apple Bananas Beer Chicken Milk Rice 0 True False True True False True 1 True False True False False True 2 True False True False False False 3 True True False False False False 4 False False True True True True 5 False False True False True True 6 False False True False True False 7 True True False False False False min_support : float (default: 0.5) A float between 0 and 1 for minumum support of the itemsets returned. The support is computed as the fraction transactions_where_item(s)_occur / total_transactions . use_colnames : bool (default: False) If True , uses the DataFrames' column names in the returned DataFrame instead of column indices. max_len : int (default: None) Maximum length of the itemsets generated. If None (default) all possible itemsets lengths (under the apriori condition) are evaluated. verbose : int (default: 0) Shows the number of iterations if >= 1 and low_memory is True . If =1 and low_memory is False , shows the number of combinations. low_memory : bool (default: False) If True , uses an iterator to search for combinations above min_support . Note that while low_memory=True should only be used for large dataset if memory resources are limited, because this implementation is approx. 3-6x slower than the default. Returns pandas DataFrame with columns ['support', 'itemsets'] of all itemsets that are >= min_support and < than max_len (if max_len is not None). Each itemset in the 'itemsets' column is of type frozenset , which is a Python built-in type that behaves similarly to sets except that it is immutable (For more info, see https://docs.python.org/3.6/library/stdtypes.html#frozenset). Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/frequent_patterns/apriori/","title":"Apriori"},{"location":"user_guide/frequent_patterns/apriori/#apriori-frequent-itemsets-via-the-apriori-algorithm","text":"Apriori function to extract frequent itemsets for association rule mining from mlxtend.frequent_patterns import apriori","title":"apriori: Frequent itemsets via the Apriori algorithm"},{"location":"user_guide/frequent_patterns/apriori/#overview","text":"Apriori is a popular algorithm [1] for extracting frequent itemsets with applications in association rule learning. The apriori algorithm has been designed to operate on databases containing transactions, such as purchases by customers of a store. An itemset is considered as \"frequent\" if it meets a user-specified support threshold. For instance, if the support threshold is set to 0.5 (50%), a frequent itemset is defined as a set of items that occur together in at least 50% of all transactions in the database.","title":"Overview"},{"location":"user_guide/frequent_patterns/apriori/#references","text":"[1] Agrawal, Rakesh, and Ramakrishnan Srikant. \" Fast algorithms for mining association rules .\" Proc. 20th int. conf. very large data bases, VLDB. Vol. 1215. 1994.","title":"References"},{"location":"user_guide/frequent_patterns/apriori/#related","text":"FP-Growth FP-Max","title":"Related"},{"location":"user_guide/frequent_patterns/apriori/#example-1-generating-frequent-itemsets","text":"The apriori function expects data in a one-hot encoded pandas DataFrame. Suppose we have the following transaction data: dataset = [['Milk', 'Onion', 'Nutmeg', 'Kidney Beans', 'Eggs', 'Yogurt'], ['Dill', 'Onion', 'Nutmeg', 'Kidney Beans', 'Eggs', 'Yogurt'], ['Milk', 'Apple', 'Kidney Beans', 'Eggs'], ['Milk', 'Unicorn', 'Corn', 'Kidney Beans', 'Yogurt'], ['Corn', 'Onion', 'Onion', 'Kidney Beans', 'Ice cream', 'Eggs']] We can transform it into the right format via the TransactionEncoder as follows: import pandas as pd from mlxtend.preprocessing import TransactionEncoder te = TransactionEncoder() te_ary = te.fit(dataset).transform(dataset) df = pd.DataFrame(te_ary, columns=te.columns_) df .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } Apple Corn Dill Eggs Ice cream Kidney Beans Milk Nutmeg Onion Unicorn Yogurt 0 False False False True False True True True True False True 1 False False True True False True False True True False True 2 True False False True False True True False False False False 3 False True False False False True True False False True True 4 False True False True True True False False True False False Now, let us return the items and itemsets with at least 60% support: from mlxtend.frequent_patterns import apriori apriori(df, min_support=0.6) .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } support itemsets 0 0.8 (3) 1 1.0 (5) 2 0.6 (6) 3 0.6 (8) 4 0.6 (10) 5 0.8 (3, 5) 6 0.6 (8, 3) 7 0.6 (5, 6) 8 0.6 (8, 5) 9 0.6 (10, 5) 10 0.6 (8, 3, 5) By default, apriori returns the column indices of the items, which may be useful in downstream operations such as association rule mining. For better readability, we can set use_colnames=True to convert these integer values into the respective item names: apriori(df, min_support=0.6, use_colnames=True) .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } support itemsets 0 0.8 (Eggs) 1 1.0 (Kidney Beans) 2 0.6 (Milk) 3 0.6 (Onion) 4 0.6 (Yogurt) 5 0.8 (Eggs, Kidney Beans) 6 0.6 (Eggs, Onion) 7 0.6 (Kidney Beans, Milk) 8 0.6 (Kidney Beans, Onion) 9 0.6 (Yogurt, Kidney Beans) 10 0.6 (Kidney Beans, Eggs, Onion)","title":"Example 1 -- Generating Frequent Itemsets"},{"location":"user_guide/frequent_patterns/apriori/#example-2-selecting-and-filtering-results","text":"The advantage of working with pandas DataFrames is that we can use its convenient features to filter the results. For instance, let's assume we are only interested in itemsets of length 2 that have a support of at least 80 percent. First, we create the frequent itemsets via apriori and add a new column that stores the length of each itemset: frequent_itemsets = apriori(df, min_support=0.6, use_colnames=True) frequent_itemsets['length'] = frequent_itemsets['itemsets'].apply(lambda x: len(x)) frequent_itemsets .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } support itemsets length 0 0.8 (Eggs) 1 1 1.0 (Kidney Beans) 1 2 0.6 (Milk) 1 3 0.6 (Onion) 1 4 0.6 (Yogurt) 1 5 0.8 (Eggs, Kidney Beans) 2 6 0.6 (Eggs, Onion) 2 7 0.6 (Kidney Beans, Milk) 2 8 0.6 (Kidney Beans, Onion) 2 9 0.6 (Yogurt, Kidney Beans) 2 10 0.6 (Kidney Beans, Eggs, Onion) 3 Then, we can select the results that satisfy our desired criteria as follows: frequent_itemsets[ (frequent_itemsets['length'] == 2) & (frequent_itemsets['support'] >= 0.8) ] .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } support itemsets length 5 0.8 (Eggs, Kidney Beans) 2 Similarly, using the Pandas API, we can select entries based on the \"itemsets\" column: frequent_itemsets[ frequent_itemsets['itemsets'] == {'Onion', 'Eggs'} ] .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } support itemsets length 6 0.6 (Eggs, Onion) 2 Frozensets Note that the entries in the \"itemsets\" column are of type frozenset , which is built-in Python type that is similar to a Python set but immutable, which makes it more efficient for certain query or comparison operations (https://docs.python.org/3.6/library/stdtypes.html#frozenset). Since frozenset s are sets, the item order does not matter. I.e., the query frequent_itemsets[ frequent_itemsets['itemsets'] == {'Onion', 'Eggs'} ] is equivalent to any of the following three frequent_itemsets[ frequent_itemsets['itemsets'] == {'Eggs', 'Onion'} ] frequent_itemsets[ frequent_itemsets['itemsets'] == frozenset(('Eggs', 'Onion')) ] frequent_itemsets[ frequent_itemsets['itemsets'] == frozenset(('Onion', 'Eggs')) ]","title":"Example 2 -- Selecting and Filtering Results"},{"location":"user_guide/frequent_patterns/apriori/#example-3-working-with-sparse-representations","text":"To save memory, you may want to represent your transaction data in the sparse format. This is especially useful if you have lots of products and small transactions. oht_ary = te.fit(dataset).transform(dataset, sparse=True) sparse_df = pd.DataFrame.sparse.from_spmatrix(oht_ary, columns=te.columns_) sparse_df .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } Apple Corn Dill Eggs Ice cream Kidney Beans Milk Nutmeg Onion Unicorn Yogurt 0 False False False True False True True True True False True 1 False False True True False True False True True False True 2 True False False True False True True False False False False 3 False True False False False True True False False True True 4 False True False True True True False False True False False apriori(sparse_df, min_support=0.6, use_colnames=True, verbose=1) Processing 21 combinations | Sampling itemset size 3 .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } support itemsets 0 0.8 (Eggs) 1 1.0 (Kidney Beans) 2 0.6 (Milk) 3 0.6 (Onion) 4 0.6 (Yogurt) 5 0.8 (Eggs, Kidney Beans) 6 0.6 (Eggs, Onion) 7 0.6 (Kidney Beans, Milk) 8 0.6 (Kidney Beans, Onion) 9 0.6 (Yogurt, Kidney Beans) 10 0.6 (Kidney Beans, Eggs, Onion)","title":"Example 3 -- Working with Sparse Representations"},{"location":"user_guide/frequent_patterns/apriori/#api","text":"apriori(df, min_support=0.5, use_colnames=False, max_len=None, verbose=0, low_memory=False) Get frequent itemsets from a one-hot DataFrame Parameters df : pandas DataFrame pandas DataFrame the encoded format. Also supports DataFrames with sparse data; for more info, please see (https://pandas.pydata.org/pandas-docs/stable/ user_guide/sparse.html#sparse-data-structures) Please note that the old pandas SparseDataFrame format is no longer supported in mlxtend >= 0.17.2. The allowed values are either 0/1 or True/False. For example, Apple Bananas Beer Chicken Milk Rice 0 True False True True False True 1 True False True False False True 2 True False True False False False 3 True True False False False False 4 False False True True True True 5 False False True False True True 6 False False True False True False 7 True True False False False False min_support : float (default: 0.5) A float between 0 and 1 for minumum support of the itemsets returned. The support is computed as the fraction transactions_where_item(s)_occur / total_transactions . use_colnames : bool (default: False) If True , uses the DataFrames' column names in the returned DataFrame instead of column indices. max_len : int (default: None) Maximum length of the itemsets generated. If None (default) all possible itemsets lengths (under the apriori condition) are evaluated. verbose : int (default: 0) Shows the number of iterations if >= 1 and low_memory is True . If =1 and low_memory is False , shows the number of combinations. low_memory : bool (default: False) If True , uses an iterator to search for combinations above min_support . Note that while low_memory=True should only be used for large dataset if memory resources are limited, because this implementation is approx. 3-6x slower than the default. Returns pandas DataFrame with columns ['support', 'itemsets'] of all itemsets that are >= min_support and < than max_len (if max_len is not None). Each itemset in the 'itemsets' column is of type frozenset , which is a Python built-in type that behaves similarly to sets except that it is immutable (For more info, see https://docs.python.org/3.6/library/stdtypes.html#frozenset). Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/frequent_patterns/apriori/","title":"API"},{"location":"user_guide/frequent_patterns/association_rules/","text":"association_rules: Association rules generation from frequent itemsets Function to generate association rules from frequent itemsets from mlxtend.frequent_patterns import association_rules Overview Rule generation is a common task in the mining of frequent patterns. An association rule is an implication expression of the form X \\rightarrow Y , where X and Y are disjoint itemsets [1]. A more concrete example based on consumer behaviour would be \\{Diapers\\} \\rightarrow \\{Beer\\} suggesting that people who buy diapers are also likely to buy beer. To evaluate the \"interest\" of such an association rule, different metrics have been developed. The current implementation make use of the confidence and lift metrics. Metrics The currently supported metrics for evaluating association rules and setting selection thresholds are listed below. Given a rule \"A -> C\", A stands for antecedent and C stands for consequent. 'support': \\text{support}(A\\rightarrow C) = \\text{support}(A \\cup C), \\;\\;\\; \\text{range: } [0, 1] introduced in [3] The support metric is defined for itemsets, not assocication rules. The table produced by the association rule mining algorithm contains three different support metrics: 'antecedent support', 'consequent support', and 'support'. Here, 'antecedent support' computes the proportion of transactions that contain the antecedent A, and 'consequent support' computes the support for the itemset of the consequent C. The 'support' metric then computes the support of the combined itemset A \\cup C. Typically, support is used to measure the abundance or frequency (often interpreted as significance or importance) of an itemset in a database. We refer to an itemset as a \"frequent itemset\" if you support is larger than a specified minimum-support threshold. Note that in general, due to the downward closure property, all subsets of a frequent itemset are also frequent. 'confidence': \\text{confidence}(A\\rightarrow C) = \\frac{\\text{support}(A\\rightarrow C)}{\\text{support}(A)}, \\;\\;\\; \\text{range: } [0, 1] introduced in [3] The confidence of a rule A->C is the probability of seeing the consequent in a transaction given that it also contains the antecedent. Note that the metric is not symmetric or directed; for instance, the confidence for A->C is different than the confidence for C->A. The confidence is 1 (maximal) for a rule A->C if the consequent and antecedent always occur together. 'lift': \\text{lift}(A\\rightarrow C) = \\frac{\\text{confidence}(A\\rightarrow C)}{\\text{support}(C)}, \\;\\;\\; \\text{range: } [0, \\infty] introduced in [4] The lift metric is commonly used to measure how much more often the antecedent and consequent of a rule A->C occur together than we would expect if they were statistically independent. If A and C are independent, the Lift score will be exactly 1. 'leverage': \\text{levarage}(A\\rightarrow C) = \\text{support}(A\\rightarrow C) - \\text{support}(A) \\times \\text{support}(C), \\;\\;\\; \\text{range: } [-1, 1] introduced in [5] Leverage computes the difference between the observed frequency of A and C appearing together and the frequency that would be expected if A and C were independent. A leverage value of 0 indicates independence. 'conviction': \\text{conviction}(A\\rightarrow C) = \\frac{1 - \\text{support}(C)}{1 - \\text{confidence}(A\\rightarrow C)}, \\;\\;\\; \\text{range: } [0, \\infty] introduced in [6] A high conviction value means that the consequent is highly depending on the antecedent. For instance, in the case of a perfect confidence score, the denominator becomes 0 (due to 1 - 1) for which the conviction score is defined as 'inf'. Similar to lift, if items are independent, the conviction is 1. 'zhangs_metric': \\text{zhangs metric}(A\\rightarrow C) = \\frac{\\text{confidence}(A\\rightarrow C) - \\text{confidence}(A'\\rightarrow C)}{Max[ \\text{confidence}(A\\rightarrow C) , \\text{confidence}(A'\\rightarrow C)]}, \\;\\;\\; \\text{range: } [-1, 1] introduced in [7] Measures both association and dissociation. Value ranges between -1 and 1. A positive value (>0) indicates Association and negative value indicated dissociation. References [1] Tan, Steinbach, Kumar. Introduction to Data Mining. Pearson New International Edition. Harlow: Pearson Education Ltd., 2014. (pp. 327-414). [2] Michael Hahsler, https://michael.hahsler.net/research/association_rules/measures.html [3] R. Agrawal, T. Imielinski, and A. Swami. Mining associations between sets of items in large databases. In Proc. of the ACM SIGMOD Int'l Conference on Management of Data, pages 207-216, Washington D.C., May 1993 [4] S. Brin, R. Motwani, J. D. Ullman, and S. Tsur. Dynamic itemset counting and implication rules for market basket data [5] Piatetsky-Shapiro, G., Discovery, analysis, and presentation of strong rules. Knowledge Discovery in Databases, 1991: p. 229-248. [6] Sergey Brin, Rajeev Motwani, Jeffrey D. Ullman, and Shalom Turk. Dynamic itemset counting and implication rules for market basket data. In SIGMOD 1997, Proceedings ACM SIGMOD International Conference on Management of Data, pages 255-264, Tucson, Arizona, USA, May 1997 [7] Xiaowei Yan , Chengqi Zhang & Shichao Zhang (2009) CONFIDENCE METRICS FOR ASSOCIATION RULE MINING, Applied Artificial Intelligence, 23:8, 713-737 https://www.tandfonline.com/doi/pdf/10.1080/08839510903208062. Example 1 -- Generating Association Rules from Frequent Itemsets The generate_rules takes dataframes of frequent itemsets as produced by the apriori , fpgrowth , or fpmax functions in mlxtend.association . To demonstrate the usage of the generate_rules method, we first create a pandas DataFrame of frequent itemsets as generated by the fpgrowth function: import pandas as pd from mlxtend.preprocessing import TransactionEncoder from mlxtend.frequent_patterns import apriori, fpmax, fpgrowth dataset = [['Milk', 'Onion', 'Nutmeg', 'Kidney Beans', 'Eggs', 'Yogurt'], ['Dill', 'Onion', 'Nutmeg', 'Kidney Beans', 'Eggs', 'Yogurt'], ['Milk', 'Apple', 'Kidney Beans', 'Eggs'], ['Milk', 'Unicorn', 'Corn', 'Kidney Beans', 'Yogurt'], ['Corn', 'Onion', 'Onion', 'Kidney Beans', 'Ice cream', 'Eggs']] te = TransactionEncoder() te_ary = te.fit(dataset).transform(dataset) df = pd.DataFrame(te_ary, columns=te.columns_) frequent_itemsets = fpgrowth(df, min_support=0.6, use_colnames=True) ### alternatively: #frequent_itemsets = apriori(df, min_support=0.6, use_colnames=True) #frequent_itemsets = fpmax(df, min_support=0.6, use_colnames=True) frequent_itemsets .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } support itemsets 0 1.0 (Kidney Beans) 1 0.8 (Eggs) 2 0.6 (Yogurt) 3 0.6 (Onion) 4 0.6 (Milk) 5 0.8 (Kidney Beans, Eggs) 6 0.6 (Kidney Beans, Yogurt) 7 0.6 (Eggs, Onion) 8 0.6 (Kidney Beans, Onion) 9 0.6 (Eggs, Kidney Beans, Onion) 10 0.6 (Kidney Beans, Milk) The generate_rules() function allows you to (1) specify your metric of interest and (2) the according threshold. Currently implemented measures are confidence and lift . Let's say you are interested in rules derived from the frequent itemsets only if the level of confidence is above the 70 percent threshold ( min_threshold=0.7 ): from mlxtend.frequent_patterns import association_rules association_rules(frequent_itemsets, metric=\"confidence\", min_threshold=0.7) .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } antecedents consequents antecedent support consequent support support confidence lift leverage conviction zhangs_metric 0 (Kidney Beans) (Eggs) 1.0 0.8 0.8 0.80 1.00 0.00 1.0 0.0 1 (Eggs) (Kidney Beans) 0.8 1.0 0.8 1.00 1.00 0.00 inf 0.0 2 (Yogurt) (Kidney Beans) 0.6 1.0 0.6 1.00 1.00 0.00 inf 0.0 3 (Eggs) (Onion) 0.8 0.6 0.6 0.75 1.25 0.12 1.6 1.0 4 (Onion) (Eggs) 0.6 0.8 0.6 1.00 1.25 0.12 inf 0.5 5 (Onion) (Kidney Beans) 0.6 1.0 0.6 1.00 1.00 0.00 inf 0.0 6 (Kidney Beans, Eggs) (Onion) 0.8 0.6 0.6 0.75 1.25 0.12 1.6 1.0 7 (Onion, Eggs) (Kidney Beans) 0.6 1.0 0.6 1.00 1.00 0.00 inf 0.0 8 (Kidney Beans, Onion) (Eggs) 0.6 0.8 0.6 1.00 1.25 0.12 inf 0.5 9 (Eggs) (Kidney Beans, Onion) 0.8 0.6 0.6 0.75 1.25 0.12 1.6 1.0 10 (Onion) (Kidney Beans, Eggs) 0.6 0.8 0.6 1.00 1.25 0.12 inf 0.5 11 (Milk) (Kidney Beans) 0.6 1.0 0.6 1.00 1.00 0.00 inf 0.0 Example 2 -- Rule Generation and Selection Criteria If you are interested in rules according to a different metric of interest, you can simply adjust the metric and min_threshold arguments . E.g. if you are only interested in rules that have a lift score of >= 1.2, you would do the following: rules = association_rules(frequent_itemsets, metric=\"lift\", min_threshold=1.2) rules .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } antecedents consequents antecedent support consequent support support confidence lift leverage conviction zhangs_metric 0 (Eggs) (Onion) 0.8 0.6 0.6 0.75 1.25 0.12 1.6 1.0 1 (Onion) (Eggs) 0.6 0.8 0.6 1.00 1.25 0.12 inf 0.5 2 (Kidney Beans, Eggs) (Onion) 0.8 0.6 0.6 0.75 1.25 0.12 1.6 1.0 3 (Kidney Beans, Onion) (Eggs) 0.6 0.8 0.6 1.00 1.25 0.12 inf 0.5 4 (Eggs) (Kidney Beans, Onion) 0.8 0.6 0.6 0.75 1.25 0.12 1.6 1.0 5 (Onion) (Kidney Beans, Eggs) 0.6 0.8 0.6 1.00 1.25 0.12 inf 0.5 Pandas DataFrames make it easy to filter the results further. Let's say we are ony interested in rules that satisfy the following criteria: at least 2 antecedents a confidence > 0.75 a lift score > 1.2 We could compute the antecedent length as follows: rules[\"antecedent_len\"] = rules[\"antecedents\"].apply(lambda x: len(x)) rules .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } antecedents consequents antecedent support consequent support support confidence lift leverage conviction zhangs_metric antecedent_len 0 (Eggs) (Onion) 0.8 0.6 0.6 0.75 1.25 0.12 1.6 1.0 1 1 (Onion) (Eggs) 0.6 0.8 0.6 1.00 1.25 0.12 inf 0.5 1 2 (Kidney Beans, Eggs) (Onion) 0.8 0.6 0.6 0.75 1.25 0.12 1.6 1.0 2 3 (Kidney Beans, Onion) (Eggs) 0.6 0.8 0.6 1.00 1.25 0.12 inf 0.5 2 4 (Eggs) (Kidney Beans, Onion) 0.8 0.6 0.6 0.75 1.25 0.12 1.6 1.0 1 5 (Onion) (Kidney Beans, Eggs) 0.6 0.8 0.6 1.00 1.25 0.12 inf 0.5 1 Then, we can use pandas' selection syntax as shown below: rules[ (rules['antecedent_len'] >= 2) & (rules['confidence'] > 0.75) & (rules['lift'] > 1.2) ] .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } antecedents consequents antecedent support consequent support support confidence lift leverage conviction zhangs_metric antecedent_len 3 (Kidney Beans, Onion) (Eggs) 0.6 0.8 0.6 1.0 1.25 0.12 inf 0.5 2 Similarly, using the Pandas API, we can select entries based on the \"antecedents\" or \"consequents\" columns: rules[rules['antecedents'] == {'Eggs', 'Kidney Beans'}] .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } antecedents consequents antecedent support consequent support support confidence lift leverage conviction zhangs_metric antecedent_len 2 (Kidney Beans, Eggs) (Onion) 0.8 0.6 0.6 0.75 1.25 0.12 1.6 1.0 2 Frozensets Note that the entries in the \"itemsets\" column are of type frozenset , which is built-in Python type that is similar to a Python set but immutable, which makes it more efficient for certain query or comparison operations (https://docs.python.org/3.6/library/stdtypes.html#frozenset). Since frozenset s are sets, the item order does not matter. I.e., the query rules[rules['antecedents'] == {'Eggs', 'Kidney Beans'}] is equivalent to any of the following three rules[rules['antecedents'] == {'Kidney Beans', 'Eggs'}] rules[rules['antecedents'] == frozenset(('Eggs', 'Kidney Beans'))] rules[rules['antecedents'] == frozenset(('Kidney Beans', 'Eggs'))] Example 3 -- Frequent Itemsets with Incomplete Antecedent and Consequent Information Most metrics computed by association_rules depends on the consequent and antecedent support score of a given rule provided in the frequent itemset input DataFrame. Consider the following example: import pandas as pd dict = {'itemsets': [['177', '176'], ['177', '179'], ['176', '178'], ['176', '179'], ['93', '100'], ['177', '178'], ['177', '176', '178']], 'support':[0.253623, 0.253623, 0.217391, 0.217391, 0.181159, 0.108696, 0.108696]} freq_itemsets = pd.DataFrame(dict) freq_itemsets .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } itemsets support 0 [177, 176] 0.253623 1 [177, 179] 0.253623 2 [176, 178] 0.217391 3 [176, 179] 0.217391 4 [93, 100] 0.181159 5 [177, 178] 0.108696 6 [177, 176, 178] 0.108696 Note that this is a \"cropped\" DataFrame that doesn't contain the support values of the item subsets. This can create problems if we want to compute the association rule metrics for, e.g., 176 => 177 . For example, the confidence is computed as \\text{confidence}(A\\rightarrow C) = \\frac{\\text{support}(A\\rightarrow C)}{\\text{support}(A)}, \\;\\;\\; \\text{range: } [0, 1] But we do not have \\text{support}(A) . All we know about \"A\"'s support is that it is at least 0.253623. In these scenarios, where not all metric's can be computed, due to incomplete input DataFrames, you can use the support_only=True option, which will only compute the support column of a given rule that does not require as much info: \\text{support}(A\\rightarrow C) = \\text{support}(A \\cup C), \\;\\;\\; \\text{range: } [0, 1] \"NaN's\" will be assigned to all other metric columns: from mlxtend.frequent_patterns import association_rules res = association_rules(freq_itemsets, support_only=True, min_threshold=0.1) res .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } antecedents consequents antecedent support consequent support support confidence lift leverage conviction zhangs_metric 0 (176) (177) NaN NaN 0.253623 NaN NaN NaN NaN NaN 1 (177) (176) NaN NaN 0.253623 NaN NaN NaN NaN NaN 2 (179) (177) NaN NaN 0.253623 NaN NaN NaN NaN NaN 3 (177) (179) NaN NaN 0.253623 NaN NaN NaN NaN NaN 4 (178) (176) NaN NaN 0.217391 NaN NaN NaN NaN NaN 5 (176) (178) NaN NaN 0.217391 NaN NaN NaN NaN NaN 6 (179) (176) NaN NaN 0.217391 NaN NaN NaN NaN NaN 7 (176) (179) NaN NaN 0.217391 NaN NaN NaN NaN NaN 8 (100) (93) NaN NaN 0.181159 NaN NaN NaN NaN NaN 9 (93) (100) NaN NaN 0.181159 NaN NaN NaN NaN NaN 10 (178) (177) NaN NaN 0.108696 NaN NaN NaN NaN NaN 11 (177) (178) NaN NaN 0.108696 NaN NaN NaN NaN NaN 12 (178, 176) (177) NaN NaN 0.108696 NaN NaN NaN NaN NaN 13 (178, 177) (176) NaN NaN 0.108696 NaN NaN NaN NaN NaN 14 (177, 176) (178) NaN NaN 0.108696 NaN NaN NaN NaN NaN 15 (178) (177, 176) NaN NaN 0.108696 NaN NaN NaN NaN NaN 16 (176) (178, 177) NaN NaN 0.108696 NaN NaN NaN NaN NaN 17 (177) (178, 176) NaN NaN 0.108696 NaN NaN NaN NaN NaN To clean up the representation, you may want to do the following: res = res[['antecedents', 'consequents', 'support']] res .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } antecedents consequents support 0 (176) (177) 0.253623 1 (177) (176) 0.253623 2 (179) (177) 0.253623 3 (177) (179) 0.253623 4 (178) (176) 0.217391 5 (176) (178) 0.217391 6 (179) (176) 0.217391 7 (176) (179) 0.217391 8 (100) (93) 0.181159 9 (93) (100) 0.181159 10 (178) (177) 0.108696 11 (177) (178) 0.108696 12 (178, 176) (177) 0.108696 13 (178, 177) (176) 0.108696 14 (177, 176) (178) 0.108696 15 (178) (177, 176) 0.108696 16 (176) (178, 177) 0.108696 17 (177) (178, 176) 0.108696 Example 4 -- Pruning Association Rules There is no specific API for pruning. Instead, the pandas API can be used on the resulting data frame to remove individual rows. E.g., suppose we have the following rules: import pandas as pd from mlxtend.preprocessing import TransactionEncoder from mlxtend.frequent_patterns import apriori, fpmax, fpgrowth from mlxtend.frequent_patterns import association_rules dataset = [['Milk', 'Onion', 'Nutmeg', 'Kidney Beans', 'Eggs', 'Yogurt'], ['Dill', 'Onion', 'Nutmeg', 'Kidney Beans', 'Eggs', 'Yogurt'], ['Milk', 'Apple', 'Kidney Beans', 'Eggs'], ['Milk', 'Unicorn', 'Corn', 'Kidney Beans', 'Yogurt'], ['Corn', 'Onion', 'Onion', 'Kidney Beans', 'Ice cream', 'Eggs']] te = TransactionEncoder() te_ary = te.fit(dataset).transform(dataset) df = pd.DataFrame(te_ary, columns=te.columns_) frequent_itemsets = fpgrowth(df, min_support=0.6, use_colnames=True) rules = association_rules(frequent_itemsets, metric=\"lift\", min_threshold=1.2) rules .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } antecedents consequents antecedent support consequent support support confidence lift leverage conviction zhangs_metric 0 (Eggs) (Onion) 0.8 0.6 0.6 0.75 1.25 0.12 1.6 1.0 1 (Onion) (Eggs) 0.6 0.8 0.6 1.00 1.25 0.12 inf 0.5 2 (Kidney Beans, Eggs) (Onion) 0.8 0.6 0.6 0.75 1.25 0.12 1.6 1.0 3 (Kidney Beans, Onion) (Eggs) 0.6 0.8 0.6 1.00 1.25 0.12 inf 0.5 4 (Eggs) (Kidney Beans, Onion) 0.8 0.6 0.6 0.75 1.25 0.12 1.6 1.0 5 (Onion) (Kidney Beans, Eggs) 0.6 0.8 0.6 1.00 1.25 0.12 inf 0.5 and we want to remove the rule \"(Onion, Kidney Beans) -> (Eggs)\". In order to to this, we can define selection masks and remove this row as follows: antecedent_sele = rules['antecedents'] == frozenset({'Onion', 'Kidney Beans'}) # or frozenset({'Kidney Beans', 'Onion'}) consequent_sele = rules['consequents'] == frozenset({'Eggs'}) final_sele = (antecedent_sele & consequent_sele) rules.loc[ ~final_sele ] .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } antecedents consequents antecedent support consequent support support confidence lift leverage conviction zhangs_metric 0 (Eggs) (Onion) 0.8 0.6 0.6 0.75 1.25 0.12 1.6 1.0 1 (Onion) (Eggs) 0.6 0.8 0.6 1.00 1.25 0.12 inf 0.5 2 (Kidney Beans, Eggs) (Onion) 0.8 0.6 0.6 0.75 1.25 0.12 1.6 1.0 4 (Eggs) (Kidney Beans, Onion) 0.8 0.6 0.6 0.75 1.25 0.12 1.6 1.0 5 (Onion) (Kidney Beans, Eggs) 0.6 0.8 0.6 1.00 1.25 0.12 inf 0.5 API","title":"Association rules"},{"location":"user_guide/frequent_patterns/association_rules/#association_rules-association-rules-generation-from-frequent-itemsets","text":"Function to generate association rules from frequent itemsets from mlxtend.frequent_patterns import association_rules","title":"association_rules: Association rules generation from frequent itemsets"},{"location":"user_guide/frequent_patterns/association_rules/#overview","text":"Rule generation is a common task in the mining of frequent patterns. An association rule is an implication expression of the form X \\rightarrow Y , where X and Y are disjoint itemsets [1]. A more concrete example based on consumer behaviour would be \\{Diapers\\} \\rightarrow \\{Beer\\} suggesting that people who buy diapers are also likely to buy beer. To evaluate the \"interest\" of such an association rule, different metrics have been developed. The current implementation make use of the confidence and lift metrics.","title":"Overview"},{"location":"user_guide/frequent_patterns/association_rules/#metrics","text":"The currently supported metrics for evaluating association rules and setting selection thresholds are listed below. Given a rule \"A -> C\", A stands for antecedent and C stands for consequent.","title":"Metrics"},{"location":"user_guide/frequent_patterns/association_rules/#support","text":"\\text{support}(A\\rightarrow C) = \\text{support}(A \\cup C), \\;\\;\\; \\text{range: } [0, 1] introduced in [3] The support metric is defined for itemsets, not assocication rules. The table produced by the association rule mining algorithm contains three different support metrics: 'antecedent support', 'consequent support', and 'support'. Here, 'antecedent support' computes the proportion of transactions that contain the antecedent A, and 'consequent support' computes the support for the itemset of the consequent C. The 'support' metric then computes the support of the combined itemset A \\cup C. Typically, support is used to measure the abundance or frequency (often interpreted as significance or importance) of an itemset in a database. We refer to an itemset as a \"frequent itemset\" if you support is larger than a specified minimum-support threshold. Note that in general, due to the downward closure property, all subsets of a frequent itemset are also frequent.","title":"'support':"},{"location":"user_guide/frequent_patterns/association_rules/#confidence","text":"\\text{confidence}(A\\rightarrow C) = \\frac{\\text{support}(A\\rightarrow C)}{\\text{support}(A)}, \\;\\;\\; \\text{range: } [0, 1] introduced in [3] The confidence of a rule A->C is the probability of seeing the consequent in a transaction given that it also contains the antecedent. Note that the metric is not symmetric or directed; for instance, the confidence for A->C is different than the confidence for C->A. The confidence is 1 (maximal) for a rule A->C if the consequent and antecedent always occur together.","title":"'confidence':"},{"location":"user_guide/frequent_patterns/association_rules/#lift","text":"\\text{lift}(A\\rightarrow C) = \\frac{\\text{confidence}(A\\rightarrow C)}{\\text{support}(C)}, \\;\\;\\; \\text{range: } [0, \\infty] introduced in [4] The lift metric is commonly used to measure how much more often the antecedent and consequent of a rule A->C occur together than we would expect if they were statistically independent. If A and C are independent, the Lift score will be exactly 1.","title":"'lift':"},{"location":"user_guide/frequent_patterns/association_rules/#leverage","text":"\\text{levarage}(A\\rightarrow C) = \\text{support}(A\\rightarrow C) - \\text{support}(A) \\times \\text{support}(C), \\;\\;\\; \\text{range: } [-1, 1] introduced in [5] Leverage computes the difference between the observed frequency of A and C appearing together and the frequency that would be expected if A and C were independent. A leverage value of 0 indicates independence.","title":"'leverage':"},{"location":"user_guide/frequent_patterns/association_rules/#conviction","text":"\\text{conviction}(A\\rightarrow C) = \\frac{1 - \\text{support}(C)}{1 - \\text{confidence}(A\\rightarrow C)}, \\;\\;\\; \\text{range: } [0, \\infty] introduced in [6] A high conviction value means that the consequent is highly depending on the antecedent. For instance, in the case of a perfect confidence score, the denominator becomes 0 (due to 1 - 1) for which the conviction score is defined as 'inf'. Similar to lift, if items are independent, the conviction is 1.","title":"'conviction':"},{"location":"user_guide/frequent_patterns/association_rules/#zhangs_metric","text":"\\text{zhangs metric}(A\\rightarrow C) = \\frac{\\text{confidence}(A\\rightarrow C) - \\text{confidence}(A'\\rightarrow C)}{Max[ \\text{confidence}(A\\rightarrow C) , \\text{confidence}(A'\\rightarrow C)]}, \\;\\;\\; \\text{range: } [-1, 1] introduced in [7] Measures both association and dissociation. Value ranges between -1 and 1. A positive value (>0) indicates Association and negative value indicated dissociation.","title":"'zhangs_metric':"},{"location":"user_guide/frequent_patterns/association_rules/#references","text":"[1] Tan, Steinbach, Kumar. Introduction to Data Mining. Pearson New International Edition. Harlow: Pearson Education Ltd., 2014. (pp. 327-414). [2] Michael Hahsler, https://michael.hahsler.net/research/association_rules/measures.html [3] R. Agrawal, T. Imielinski, and A. Swami. Mining associations between sets of items in large databases. In Proc. of the ACM SIGMOD Int'l Conference on Management of Data, pages 207-216, Washington D.C., May 1993 [4] S. Brin, R. Motwani, J. D. Ullman, and S. Tsur. Dynamic itemset counting and implication rules for market basket data [5] Piatetsky-Shapiro, G., Discovery, analysis, and presentation of strong rules. Knowledge Discovery in Databases, 1991: p. 229-248. [6] Sergey Brin, Rajeev Motwani, Jeffrey D. Ullman, and Shalom Turk. Dynamic itemset counting and implication rules for market basket data. In SIGMOD 1997, Proceedings ACM SIGMOD International Conference on Management of Data, pages 255-264, Tucson, Arizona, USA, May 1997 [7] Xiaowei Yan , Chengqi Zhang & Shichao Zhang (2009) CONFIDENCE METRICS FOR ASSOCIATION RULE MINING, Applied Artificial Intelligence, 23:8, 713-737 https://www.tandfonline.com/doi/pdf/10.1080/08839510903208062.","title":"References"},{"location":"user_guide/frequent_patterns/association_rules/#example-1-generating-association-rules-from-frequent-itemsets","text":"The generate_rules takes dataframes of frequent itemsets as produced by the apriori , fpgrowth , or fpmax functions in mlxtend.association . To demonstrate the usage of the generate_rules method, we first create a pandas DataFrame of frequent itemsets as generated by the fpgrowth function: import pandas as pd from mlxtend.preprocessing import TransactionEncoder from mlxtend.frequent_patterns import apriori, fpmax, fpgrowth dataset = [['Milk', 'Onion', 'Nutmeg', 'Kidney Beans', 'Eggs', 'Yogurt'], ['Dill', 'Onion', 'Nutmeg', 'Kidney Beans', 'Eggs', 'Yogurt'], ['Milk', 'Apple', 'Kidney Beans', 'Eggs'], ['Milk', 'Unicorn', 'Corn', 'Kidney Beans', 'Yogurt'], ['Corn', 'Onion', 'Onion', 'Kidney Beans', 'Ice cream', 'Eggs']] te = TransactionEncoder() te_ary = te.fit(dataset).transform(dataset) df = pd.DataFrame(te_ary, columns=te.columns_) frequent_itemsets = fpgrowth(df, min_support=0.6, use_colnames=True) ### alternatively: #frequent_itemsets = apriori(df, min_support=0.6, use_colnames=True) #frequent_itemsets = fpmax(df, min_support=0.6, use_colnames=True) frequent_itemsets .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } support itemsets 0 1.0 (Kidney Beans) 1 0.8 (Eggs) 2 0.6 (Yogurt) 3 0.6 (Onion) 4 0.6 (Milk) 5 0.8 (Kidney Beans, Eggs) 6 0.6 (Kidney Beans, Yogurt) 7 0.6 (Eggs, Onion) 8 0.6 (Kidney Beans, Onion) 9 0.6 (Eggs, Kidney Beans, Onion) 10 0.6 (Kidney Beans, Milk) The generate_rules() function allows you to (1) specify your metric of interest and (2) the according threshold. Currently implemented measures are confidence and lift . Let's say you are interested in rules derived from the frequent itemsets only if the level of confidence is above the 70 percent threshold ( min_threshold=0.7 ): from mlxtend.frequent_patterns import association_rules association_rules(frequent_itemsets, metric=\"confidence\", min_threshold=0.7) .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } antecedents consequents antecedent support consequent support support confidence lift leverage conviction zhangs_metric 0 (Kidney Beans) (Eggs) 1.0 0.8 0.8 0.80 1.00 0.00 1.0 0.0 1 (Eggs) (Kidney Beans) 0.8 1.0 0.8 1.00 1.00 0.00 inf 0.0 2 (Yogurt) (Kidney Beans) 0.6 1.0 0.6 1.00 1.00 0.00 inf 0.0 3 (Eggs) (Onion) 0.8 0.6 0.6 0.75 1.25 0.12 1.6 1.0 4 (Onion) (Eggs) 0.6 0.8 0.6 1.00 1.25 0.12 inf 0.5 5 (Onion) (Kidney Beans) 0.6 1.0 0.6 1.00 1.00 0.00 inf 0.0 6 (Kidney Beans, Eggs) (Onion) 0.8 0.6 0.6 0.75 1.25 0.12 1.6 1.0 7 (Onion, Eggs) (Kidney Beans) 0.6 1.0 0.6 1.00 1.00 0.00 inf 0.0 8 (Kidney Beans, Onion) (Eggs) 0.6 0.8 0.6 1.00 1.25 0.12 inf 0.5 9 (Eggs) (Kidney Beans, Onion) 0.8 0.6 0.6 0.75 1.25 0.12 1.6 1.0 10 (Onion) (Kidney Beans, Eggs) 0.6 0.8 0.6 1.00 1.25 0.12 inf 0.5 11 (Milk) (Kidney Beans) 0.6 1.0 0.6 1.00 1.00 0.00 inf 0.0","title":"Example 1 -- Generating Association Rules from Frequent Itemsets"},{"location":"user_guide/frequent_patterns/association_rules/#example-2-rule-generation-and-selection-criteria","text":"If you are interested in rules according to a different metric of interest, you can simply adjust the metric and min_threshold arguments . E.g. if you are only interested in rules that have a lift score of >= 1.2, you would do the following: rules = association_rules(frequent_itemsets, metric=\"lift\", min_threshold=1.2) rules .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } antecedents consequents antecedent support consequent support support confidence lift leverage conviction zhangs_metric 0 (Eggs) (Onion) 0.8 0.6 0.6 0.75 1.25 0.12 1.6 1.0 1 (Onion) (Eggs) 0.6 0.8 0.6 1.00 1.25 0.12 inf 0.5 2 (Kidney Beans, Eggs) (Onion) 0.8 0.6 0.6 0.75 1.25 0.12 1.6 1.0 3 (Kidney Beans, Onion) (Eggs) 0.6 0.8 0.6 1.00 1.25 0.12 inf 0.5 4 (Eggs) (Kidney Beans, Onion) 0.8 0.6 0.6 0.75 1.25 0.12 1.6 1.0 5 (Onion) (Kidney Beans, Eggs) 0.6 0.8 0.6 1.00 1.25 0.12 inf 0.5 Pandas DataFrames make it easy to filter the results further. Let's say we are ony interested in rules that satisfy the following criteria: at least 2 antecedents a confidence > 0.75 a lift score > 1.2 We could compute the antecedent length as follows: rules[\"antecedent_len\"] = rules[\"antecedents\"].apply(lambda x: len(x)) rules .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } antecedents consequents antecedent support consequent support support confidence lift leverage conviction zhangs_metric antecedent_len 0 (Eggs) (Onion) 0.8 0.6 0.6 0.75 1.25 0.12 1.6 1.0 1 1 (Onion) (Eggs) 0.6 0.8 0.6 1.00 1.25 0.12 inf 0.5 1 2 (Kidney Beans, Eggs) (Onion) 0.8 0.6 0.6 0.75 1.25 0.12 1.6 1.0 2 3 (Kidney Beans, Onion) (Eggs) 0.6 0.8 0.6 1.00 1.25 0.12 inf 0.5 2 4 (Eggs) (Kidney Beans, Onion) 0.8 0.6 0.6 0.75 1.25 0.12 1.6 1.0 1 5 (Onion) (Kidney Beans, Eggs) 0.6 0.8 0.6 1.00 1.25 0.12 inf 0.5 1 Then, we can use pandas' selection syntax as shown below: rules[ (rules['antecedent_len'] >= 2) & (rules['confidence'] > 0.75) & (rules['lift'] > 1.2) ] .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } antecedents consequents antecedent support consequent support support confidence lift leverage conviction zhangs_metric antecedent_len 3 (Kidney Beans, Onion) (Eggs) 0.6 0.8 0.6 1.0 1.25 0.12 inf 0.5 2 Similarly, using the Pandas API, we can select entries based on the \"antecedents\" or \"consequents\" columns: rules[rules['antecedents'] == {'Eggs', 'Kidney Beans'}] .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } antecedents consequents antecedent support consequent support support confidence lift leverage conviction zhangs_metric antecedent_len 2 (Kidney Beans, Eggs) (Onion) 0.8 0.6 0.6 0.75 1.25 0.12 1.6 1.0 2 Frozensets Note that the entries in the \"itemsets\" column are of type frozenset , which is built-in Python type that is similar to a Python set but immutable, which makes it more efficient for certain query or comparison operations (https://docs.python.org/3.6/library/stdtypes.html#frozenset). Since frozenset s are sets, the item order does not matter. I.e., the query rules[rules['antecedents'] == {'Eggs', 'Kidney Beans'}] is equivalent to any of the following three rules[rules['antecedents'] == {'Kidney Beans', 'Eggs'}] rules[rules['antecedents'] == frozenset(('Eggs', 'Kidney Beans'))] rules[rules['antecedents'] == frozenset(('Kidney Beans', 'Eggs'))]","title":"Example 2 -- Rule Generation and Selection Criteria"},{"location":"user_guide/frequent_patterns/association_rules/#example-3-frequent-itemsets-with-incomplete-antecedent-and-consequent-information","text":"Most metrics computed by association_rules depends on the consequent and antecedent support score of a given rule provided in the frequent itemset input DataFrame. Consider the following example: import pandas as pd dict = {'itemsets': [['177', '176'], ['177', '179'], ['176', '178'], ['176', '179'], ['93', '100'], ['177', '178'], ['177', '176', '178']], 'support':[0.253623, 0.253623, 0.217391, 0.217391, 0.181159, 0.108696, 0.108696]} freq_itemsets = pd.DataFrame(dict) freq_itemsets .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } itemsets support 0 [177, 176] 0.253623 1 [177, 179] 0.253623 2 [176, 178] 0.217391 3 [176, 179] 0.217391 4 [93, 100] 0.181159 5 [177, 178] 0.108696 6 [177, 176, 178] 0.108696 Note that this is a \"cropped\" DataFrame that doesn't contain the support values of the item subsets. This can create problems if we want to compute the association rule metrics for, e.g., 176 => 177 . For example, the confidence is computed as \\text{confidence}(A\\rightarrow C) = \\frac{\\text{support}(A\\rightarrow C)}{\\text{support}(A)}, \\;\\;\\; \\text{range: } [0, 1] But we do not have \\text{support}(A) . All we know about \"A\"'s support is that it is at least 0.253623. In these scenarios, where not all metric's can be computed, due to incomplete input DataFrames, you can use the support_only=True option, which will only compute the support column of a given rule that does not require as much info: \\text{support}(A\\rightarrow C) = \\text{support}(A \\cup C), \\;\\;\\; \\text{range: } [0, 1] \"NaN's\" will be assigned to all other metric columns: from mlxtend.frequent_patterns import association_rules res = association_rules(freq_itemsets, support_only=True, min_threshold=0.1) res .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } antecedents consequents antecedent support consequent support support confidence lift leverage conviction zhangs_metric 0 (176) (177) NaN NaN 0.253623 NaN NaN NaN NaN NaN 1 (177) (176) NaN NaN 0.253623 NaN NaN NaN NaN NaN 2 (179) (177) NaN NaN 0.253623 NaN NaN NaN NaN NaN 3 (177) (179) NaN NaN 0.253623 NaN NaN NaN NaN NaN 4 (178) (176) NaN NaN 0.217391 NaN NaN NaN NaN NaN 5 (176) (178) NaN NaN 0.217391 NaN NaN NaN NaN NaN 6 (179) (176) NaN NaN 0.217391 NaN NaN NaN NaN NaN 7 (176) (179) NaN NaN 0.217391 NaN NaN NaN NaN NaN 8 (100) (93) NaN NaN 0.181159 NaN NaN NaN NaN NaN 9 (93) (100) NaN NaN 0.181159 NaN NaN NaN NaN NaN 10 (178) (177) NaN NaN 0.108696 NaN NaN NaN NaN NaN 11 (177) (178) NaN NaN 0.108696 NaN NaN NaN NaN NaN 12 (178, 176) (177) NaN NaN 0.108696 NaN NaN NaN NaN NaN 13 (178, 177) (176) NaN NaN 0.108696 NaN NaN NaN NaN NaN 14 (177, 176) (178) NaN NaN 0.108696 NaN NaN NaN NaN NaN 15 (178) (177, 176) NaN NaN 0.108696 NaN NaN NaN NaN NaN 16 (176) (178, 177) NaN NaN 0.108696 NaN NaN NaN NaN NaN 17 (177) (178, 176) NaN NaN 0.108696 NaN NaN NaN NaN NaN To clean up the representation, you may want to do the following: res = res[['antecedents', 'consequents', 'support']] res .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } antecedents consequents support 0 (176) (177) 0.253623 1 (177) (176) 0.253623 2 (179) (177) 0.253623 3 (177) (179) 0.253623 4 (178) (176) 0.217391 5 (176) (178) 0.217391 6 (179) (176) 0.217391 7 (176) (179) 0.217391 8 (100) (93) 0.181159 9 (93) (100) 0.181159 10 (178) (177) 0.108696 11 (177) (178) 0.108696 12 (178, 176) (177) 0.108696 13 (178, 177) (176) 0.108696 14 (177, 176) (178) 0.108696 15 (178) (177, 176) 0.108696 16 (176) (178, 177) 0.108696 17 (177) (178, 176) 0.108696","title":"Example 3 -- Frequent Itemsets with Incomplete Antecedent and Consequent Information"},{"location":"user_guide/frequent_patterns/association_rules/#example-4-pruning-association-rules","text":"There is no specific API for pruning. Instead, the pandas API can be used on the resulting data frame to remove individual rows. E.g., suppose we have the following rules: import pandas as pd from mlxtend.preprocessing import TransactionEncoder from mlxtend.frequent_patterns import apriori, fpmax, fpgrowth from mlxtend.frequent_patterns import association_rules dataset = [['Milk', 'Onion', 'Nutmeg', 'Kidney Beans', 'Eggs', 'Yogurt'], ['Dill', 'Onion', 'Nutmeg', 'Kidney Beans', 'Eggs', 'Yogurt'], ['Milk', 'Apple', 'Kidney Beans', 'Eggs'], ['Milk', 'Unicorn', 'Corn', 'Kidney Beans', 'Yogurt'], ['Corn', 'Onion', 'Onion', 'Kidney Beans', 'Ice cream', 'Eggs']] te = TransactionEncoder() te_ary = te.fit(dataset).transform(dataset) df = pd.DataFrame(te_ary, columns=te.columns_) frequent_itemsets = fpgrowth(df, min_support=0.6, use_colnames=True) rules = association_rules(frequent_itemsets, metric=\"lift\", min_threshold=1.2) rules .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } antecedents consequents antecedent support consequent support support confidence lift leverage conviction zhangs_metric 0 (Eggs) (Onion) 0.8 0.6 0.6 0.75 1.25 0.12 1.6 1.0 1 (Onion) (Eggs) 0.6 0.8 0.6 1.00 1.25 0.12 inf 0.5 2 (Kidney Beans, Eggs) (Onion) 0.8 0.6 0.6 0.75 1.25 0.12 1.6 1.0 3 (Kidney Beans, Onion) (Eggs) 0.6 0.8 0.6 1.00 1.25 0.12 inf 0.5 4 (Eggs) (Kidney Beans, Onion) 0.8 0.6 0.6 0.75 1.25 0.12 1.6 1.0 5 (Onion) (Kidney Beans, Eggs) 0.6 0.8 0.6 1.00 1.25 0.12 inf 0.5 and we want to remove the rule \"(Onion, Kidney Beans) -> (Eggs)\". In order to to this, we can define selection masks and remove this row as follows: antecedent_sele = rules['antecedents'] == frozenset({'Onion', 'Kidney Beans'}) # or frozenset({'Kidney Beans', 'Onion'}) consequent_sele = rules['consequents'] == frozenset({'Eggs'}) final_sele = (antecedent_sele & consequent_sele) rules.loc[ ~final_sele ] .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } antecedents consequents antecedent support consequent support support confidence lift leverage conviction zhangs_metric 0 (Eggs) (Onion) 0.8 0.6 0.6 0.75 1.25 0.12 1.6 1.0 1 (Onion) (Eggs) 0.6 0.8 0.6 1.00 1.25 0.12 inf 0.5 2 (Kidney Beans, Eggs) (Onion) 0.8 0.6 0.6 0.75 1.25 0.12 1.6 1.0 4 (Eggs) (Kidney Beans, Onion) 0.8 0.6 0.6 0.75 1.25 0.12 1.6 1.0 5 (Onion) (Kidney Beans, Eggs) 0.6 0.8 0.6 1.00 1.25 0.12 inf 0.5","title":"Example 4 -- Pruning Association Rules"},{"location":"user_guide/frequent_patterns/association_rules/#api","text":"","title":"API"},{"location":"user_guide/frequent_patterns/fpgrowth/","text":"fpgrowth: Frequent itemsets via the FP-growth algorithm Function implementing FP-Growth to extract frequent itemsets for association rule mining from mlxtend.frequent_patterns import fpgrowth Overview FP-Growth [1] is an algorithm for extracting frequent itemsets with applications in association rule learning that emerged as a popular alternative to the established Apriori algorithm [2]. In general, the algorithm has been designed to operate on databases containing transactions, such as purchases by customers of a store. An itemset is considered as \"frequent\" if it meets a user-specified support threshold. For instance, if the support threshold is set to 0.5 (50%), a frequent itemset is defined as a set of items that occur together in at least 50% of all transactions in the database. In particular, and what makes it different from the Apriori frequent pattern mining algorithm, FP-Growth is an frequent pattern mining algorithm that does not require candidate generation. Internally, it uses a so-called FP-tree (frequent pattern tree) datastrucure without generating the candidate sets explicitly, which makes it particularly attractive for large datasets. References [1] Han, Jiawei, Jian Pei, Yiwen Yin, and Runying Mao. \"Mining frequent patterns without candidate generation. \" A frequent-pattern tree approach. \" Data mining and knowledge discovery 8, no. 1 (2004): 53-87. [2] Agrawal, Rakesh, and Ramakrishnan Srikant. \" Fast algorithms for mining association rules .\" Proc. 20th int. conf. very large data bases, VLDB. Vol. 1215. 1994. Related FP-Max Apriori Example 1 -- Generating Frequent Itemsets The fpgrowth function expects data in a one-hot encoded pandas DataFrame. Suppose we have the following transaction data: dataset = [['Milk', 'Onion', 'Nutmeg', 'Kidney Beans', 'Eggs', 'Yogurt'], ['Dill', 'Onion', 'Nutmeg', 'Kidney Beans', 'Eggs', 'Yogurt'], ['Milk', 'Apple', 'Kidney Beans', 'Eggs'], ['Milk', 'Unicorn', 'Corn', 'Kidney Beans', 'Yogurt'], ['Corn', 'Onion', 'Onion', 'Kidney Beans', 'Ice cream', 'Eggs']] We can transform it into the right format via the TransactionEncoder as follows: import pandas as pd from mlxtend.preprocessing import TransactionEncoder te = TransactionEncoder() te_ary = te.fit(dataset).transform(dataset) df = pd.DataFrame(te_ary, columns=te.columns_) df .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } Apple Corn Dill Eggs Ice cream Kidney Beans Milk Nutmeg Onion Unicorn Yogurt 0 False False False True False True True True True False True 1 False False True True False True False True True False True 2 True False False True False True True False False False False 3 False True False False False True True False False True True 4 False True False True True True False False True False False Now, let us return the items and itemsets with at least 60% support: from mlxtend.frequent_patterns import fpgrowth fpgrowth(df, min_support=0.6) .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } support itemsets 0 1.0 (5) 1 0.8 (3) 2 0.6 (10) 3 0.6 (8) 4 0.6 (6) 5 0.8 (3, 5) 6 0.6 (10, 5) 7 0.6 (8, 3) 8 0.6 (8, 5) 9 0.6 (8, 3, 5) 10 0.6 (5, 6) By default, fpgrowth returns the column indices of the items, which may be useful in downstream operations such as association rule mining. For better readability, we can set use_colnames=True to convert these integer values into the respective item names: fpgrowth(df, min_support=0.6, use_colnames=True) .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } support itemsets 0 1.0 (Kidney Beans) 1 0.8 (Eggs) 2 0.6 (Yogurt) 3 0.6 (Onion) 4 0.6 (Milk) 5 0.8 (Eggs, Kidney Beans) 6 0.6 (Yogurt, Kidney Beans) 7 0.6 (Eggs, Onion) 8 0.6 (Onion, Kidney Beans) 9 0.6 (Eggs, Onion, Kidney Beans) 10 0.6 (Milk, Kidney Beans) Example 2 -- Apriori versus FPGrowth Since FP-Growth doesn't require creating candidate sets explicitly, it can be magnitudes faster than the alternative Apriori algorithm. For instance, the following cells compare the performance of the Apriori algorithm to the performance of FP-Growth -- even in this very simple toy dataset scenario, FP-Growth is about 5 times faster. import pandas as pd from mlxtend.preprocessing import TransactionEncoder te = TransactionEncoder() te_ary = te.fit(dataset).transform(dataset) df = pd.DataFrame(te_ary, columns=te.columns_) from mlxtend.frequent_patterns import apriori %timeit -n 100 -r 10 apriori(df, min_support=0.6) 850 \u00b5s \u00b1 39.3 \u00b5s per loop (mean \u00b1 std. dev. of 10 runs, 100 loops each) %timeit -n 100 -r 10 apriori(df, min_support=0.6, low_memory=True) 941 \u00b5s \u00b1 30.6 \u00b5s per loop (mean \u00b1 std. dev. of 10 runs, 100 loops each) from mlxtend.frequent_patterns import fpgrowth %timeit -n 100 -r 10 fpgrowth(df, min_support=0.6) 320 \u00b5s \u00b1 9.21 \u00b5s per loop (mean \u00b1 std. dev. of 10 runs, 100 loops each) More Examples Please note that since the fpgrowth function is a drop-in replacement for apriori , it comes with the same set of function arguments and return arguments. Thus, for more examples, please see the apriori documentation. API fpgrowth(df, min_support=0.5, use_colnames=False, max_len=None, verbose=0) Get frequent itemsets from a one-hot DataFrame Parameters df : pandas DataFrame pandas DataFrame the encoded format. Also supports DataFrames with sparse data; for more info, please see https://pandas.pydata.org/pandas-docs/stable/user_guide/sparse.html#sparse-data-structures. Please note that the old pandas SparseDataFrame format is no longer supported in mlxtend >= 0.17.2. The allowed values are either 0/1 or True/False. For example, Apple Bananas Beer Chicken Milk Rice 0 True False True True False True 1 True False True False False True 2 True False True False False False 3 True True False False False False 4 False False True True True True 5 False False True False True True 6 False False True False True False 7 True True False False False False min_support : float (default: 0.5) A float between 0 and 1 for minimum support of the itemsets returned. The support is computed as the fraction transactions_where_item(s)_occur / total_transactions. use_colnames : bool (default: False) If true, uses the DataFrames' column names in the returned DataFrame instead of column indices. max_len : int (default: None) Maximum length of the itemsets generated. If None (default) all possible itemsets lengths are evaluated. verbose : int (default: 0) Shows the stages of conditional tree generation. Returns pandas DataFrame with columns ['support', 'itemsets'] of all itemsets that are >= min_support and < than max_len (if max_len is not None). Each itemset in the 'itemsets' column is of type frozenset , which is a Python built-in type that behaves similarly to sets except that it is immutable (For more info, see https://docs.python.org/3.6/library/stdtypes.html#frozenset). Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/frequent_patterns/fpgrowth/","title":"Fpgrowth"},{"location":"user_guide/frequent_patterns/fpgrowth/#fpgrowth-frequent-itemsets-via-the-fp-growth-algorithm","text":"Function implementing FP-Growth to extract frequent itemsets for association rule mining from mlxtend.frequent_patterns import fpgrowth","title":"fpgrowth: Frequent itemsets via the FP-growth algorithm"},{"location":"user_guide/frequent_patterns/fpgrowth/#overview","text":"FP-Growth [1] is an algorithm for extracting frequent itemsets with applications in association rule learning that emerged as a popular alternative to the established Apriori algorithm [2]. In general, the algorithm has been designed to operate on databases containing transactions, such as purchases by customers of a store. An itemset is considered as \"frequent\" if it meets a user-specified support threshold. For instance, if the support threshold is set to 0.5 (50%), a frequent itemset is defined as a set of items that occur together in at least 50% of all transactions in the database. In particular, and what makes it different from the Apriori frequent pattern mining algorithm, FP-Growth is an frequent pattern mining algorithm that does not require candidate generation. Internally, it uses a so-called FP-tree (frequent pattern tree) datastrucure without generating the candidate sets explicitly, which makes it particularly attractive for large datasets.","title":"Overview"},{"location":"user_guide/frequent_patterns/fpgrowth/#references","text":"[1] Han, Jiawei, Jian Pei, Yiwen Yin, and Runying Mao. \"Mining frequent patterns without candidate generation. \" A frequent-pattern tree approach. \" Data mining and knowledge discovery 8, no. 1 (2004): 53-87. [2] Agrawal, Rakesh, and Ramakrishnan Srikant. \" Fast algorithms for mining association rules .\" Proc. 20th int. conf. very large data bases, VLDB. Vol. 1215. 1994.","title":"References"},{"location":"user_guide/frequent_patterns/fpgrowth/#related","text":"FP-Max Apriori","title":"Related"},{"location":"user_guide/frequent_patterns/fpgrowth/#example-1-generating-frequent-itemsets","text":"The fpgrowth function expects data in a one-hot encoded pandas DataFrame. Suppose we have the following transaction data: dataset = [['Milk', 'Onion', 'Nutmeg', 'Kidney Beans', 'Eggs', 'Yogurt'], ['Dill', 'Onion', 'Nutmeg', 'Kidney Beans', 'Eggs', 'Yogurt'], ['Milk', 'Apple', 'Kidney Beans', 'Eggs'], ['Milk', 'Unicorn', 'Corn', 'Kidney Beans', 'Yogurt'], ['Corn', 'Onion', 'Onion', 'Kidney Beans', 'Ice cream', 'Eggs']] We can transform it into the right format via the TransactionEncoder as follows: import pandas as pd from mlxtend.preprocessing import TransactionEncoder te = TransactionEncoder() te_ary = te.fit(dataset).transform(dataset) df = pd.DataFrame(te_ary, columns=te.columns_) df .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } Apple Corn Dill Eggs Ice cream Kidney Beans Milk Nutmeg Onion Unicorn Yogurt 0 False False False True False True True True True False True 1 False False True True False True False True True False True 2 True False False True False True True False False False False 3 False True False False False True True False False True True 4 False True False True True True False False True False False Now, let us return the items and itemsets with at least 60% support: from mlxtend.frequent_patterns import fpgrowth fpgrowth(df, min_support=0.6) .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } support itemsets 0 1.0 (5) 1 0.8 (3) 2 0.6 (10) 3 0.6 (8) 4 0.6 (6) 5 0.8 (3, 5) 6 0.6 (10, 5) 7 0.6 (8, 3) 8 0.6 (8, 5) 9 0.6 (8, 3, 5) 10 0.6 (5, 6) By default, fpgrowth returns the column indices of the items, which may be useful in downstream operations such as association rule mining. For better readability, we can set use_colnames=True to convert these integer values into the respective item names: fpgrowth(df, min_support=0.6, use_colnames=True) .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } support itemsets 0 1.0 (Kidney Beans) 1 0.8 (Eggs) 2 0.6 (Yogurt) 3 0.6 (Onion) 4 0.6 (Milk) 5 0.8 (Eggs, Kidney Beans) 6 0.6 (Yogurt, Kidney Beans) 7 0.6 (Eggs, Onion) 8 0.6 (Onion, Kidney Beans) 9 0.6 (Eggs, Onion, Kidney Beans) 10 0.6 (Milk, Kidney Beans)","title":"Example 1 -- Generating Frequent Itemsets"},{"location":"user_guide/frequent_patterns/fpgrowth/#example-2-apriori-versus-fpgrowth","text":"Since FP-Growth doesn't require creating candidate sets explicitly, it can be magnitudes faster than the alternative Apriori algorithm. For instance, the following cells compare the performance of the Apriori algorithm to the performance of FP-Growth -- even in this very simple toy dataset scenario, FP-Growth is about 5 times faster. import pandas as pd from mlxtend.preprocessing import TransactionEncoder te = TransactionEncoder() te_ary = te.fit(dataset).transform(dataset) df = pd.DataFrame(te_ary, columns=te.columns_) from mlxtend.frequent_patterns import apriori %timeit -n 100 -r 10 apriori(df, min_support=0.6) 850 \u00b5s \u00b1 39.3 \u00b5s per loop (mean \u00b1 std. dev. of 10 runs, 100 loops each) %timeit -n 100 -r 10 apriori(df, min_support=0.6, low_memory=True) 941 \u00b5s \u00b1 30.6 \u00b5s per loop (mean \u00b1 std. dev. of 10 runs, 100 loops each) from mlxtend.frequent_patterns import fpgrowth %timeit -n 100 -r 10 fpgrowth(df, min_support=0.6) 320 \u00b5s \u00b1 9.21 \u00b5s per loop (mean \u00b1 std. dev. of 10 runs, 100 loops each)","title":"Example 2 -- Apriori versus FPGrowth"},{"location":"user_guide/frequent_patterns/fpgrowth/#more-examples","text":"Please note that since the fpgrowth function is a drop-in replacement for apriori , it comes with the same set of function arguments and return arguments. Thus, for more examples, please see the apriori documentation.","title":"More Examples"},{"location":"user_guide/frequent_patterns/fpgrowth/#api","text":"fpgrowth(df, min_support=0.5, use_colnames=False, max_len=None, verbose=0) Get frequent itemsets from a one-hot DataFrame Parameters df : pandas DataFrame pandas DataFrame the encoded format. Also supports DataFrames with sparse data; for more info, please see https://pandas.pydata.org/pandas-docs/stable/user_guide/sparse.html#sparse-data-structures. Please note that the old pandas SparseDataFrame format is no longer supported in mlxtend >= 0.17.2. The allowed values are either 0/1 or True/False. For example, Apple Bananas Beer Chicken Milk Rice 0 True False True True False True 1 True False True False False True 2 True False True False False False 3 True True False False False False 4 False False True True True True 5 False False True False True True 6 False False True False True False 7 True True False False False False min_support : float (default: 0.5) A float between 0 and 1 for minimum support of the itemsets returned. The support is computed as the fraction transactions_where_item(s)_occur / total_transactions. use_colnames : bool (default: False) If true, uses the DataFrames' column names in the returned DataFrame instead of column indices. max_len : int (default: None) Maximum length of the itemsets generated. If None (default) all possible itemsets lengths are evaluated. verbose : int (default: 0) Shows the stages of conditional tree generation. Returns pandas DataFrame with columns ['support', 'itemsets'] of all itemsets that are >= min_support and < than max_len (if max_len is not None). Each itemset in the 'itemsets' column is of type frozenset , which is a Python built-in type that behaves similarly to sets except that it is immutable (For more info, see https://docs.python.org/3.6/library/stdtypes.html#frozenset). Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/frequent_patterns/fpgrowth/","title":"API"},{"location":"user_guide/frequent_patterns/fpmax/","text":"fpmax: Maximal itemsets via the FP-Max algorithm Function implementing FP-Max to extract maximal itemsets for association rule mining from mlxtend.frequent_patterns import fpmax Overview The Apriori algorithm is among the first and most popular algorithms for frequent itemset generation (frequent itemsets are then used for association rule mining). However, the runtime of Apriori can be quite large, especially for datasets with a large number of unique items, as the runtime grows exponentially depending on the number of unique items. In contrast to Apriori, FP-Growth is a frequent pattern generation algorithm that inserts items into a pattern search tree, which allows it to have a linear increase in runtime with respect to the number of unique items or entries. FP-Max is a variant of FP-Growth, which focuses on obtaining maximal itemsets. An itemset X is said to maximal if X is frequent and there exists no frequent super-pattern containing X. In other words, a frequent pattern X cannot be sub-pattern of larger frequent pattern to qualify for the definition maximal itemset . References [1] Grahne, G., & Zhu, J. (2003, November). Efficiently using prefix-trees in mining frequent itemsets. In FIMI (Vol. 90). Related FP-Growth Apriori Example 1 -- Maximal Itemsets The fpmax function expects data in a one-hot encoded pandas DataFrame. Suppose we have the following transaction data: dataset = [['Milk', 'Onion', 'Nutmeg', 'Kidney Beans', 'Eggs', 'Yogurt'], ['Dill', 'Onion', 'Nutmeg', 'Kidney Beans', 'Eggs', 'Yogurt'], ['Milk', 'Apple', 'Kidney Beans', 'Eggs'], ['Milk', 'Unicorn', 'Corn', 'Kidney Beans', 'Yogurt'], ['Corn', 'Onion', 'Onion', 'Kidney Beans', 'Ice cream', 'Eggs']] We can transform it into the right format via the TransactionEncoder as follows: import pandas as pd from mlxtend.preprocessing import TransactionEncoder te = TransactionEncoder() te_ary = te.fit(dataset).transform(dataset) df = pd.DataFrame(te_ary, columns=te.columns_) df .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } Apple Corn Dill Eggs Ice cream Kidney Beans Milk Nutmeg Onion Unicorn Yogurt 0 False False False True False True True True True False True 1 False False True True False True False True True False True 2 True False False True False True True False False False False 3 False True False False False True True False False True True 4 False True False True True True False False True False False Now, let us return the items and itemsets with at least 60% support: from mlxtend.frequent_patterns import fpmax fpmax(df, min_support=0.6) .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } support itemsets 0 0.6 (5, 6) 1 0.6 (8, 3, 5) 2 0.6 (10, 5) By default, fpmax returns the column indices of the items, which may be useful in downstream operations such as association rule mining. For better readability, we can set use_colnames=True to convert these integer values into the respective item names: fpmax(df, min_support=0.6, use_colnames=True) .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } support itemsets 0 0.6 (Kidney Beans, Milk) 1 0.6 (Onion, Eggs, Kidney Beans) 2 0.6 (Kidney Beans, Yogurt) More Examples Please note that since the fpmax function is a drop-in replacement for fpgrowth and apriori , it comes with the same set of function arguments and return arguments. Thus, for more examples, please see the apriori documentation. API fpmax(df, min_support=0.5, use_colnames=False, max_len=None, verbose=0) Get maximal frequent itemsets from a one-hot DataFrame Parameters df : pandas DataFrame pandas DataFrame the encoded format. Also supports DataFrames with sparse data; for more info, please see (https://pandas.pydata.org/pandas-docs/stable/ user_guide/sparse.html#sparse-data-structures) Please note that the old pandas SparseDataFrame format is no longer supported in mlxtend >= 0.17.2. The allowed values are either 0/1 or True/False. For example, Apple Bananas Beer Chicken Milk Rice 0 True False True True False True 1 True False True False False True 2 True False True False False False 3 True True False False False False 4 False False True True True True 5 False False True False True True 6 False False True False True False 7 True True False False False False min_support : float (default: 0.5) A float between 0 and 1 for minimum support of the itemsets returned. The support is computed as the fraction transactions_where_item(s)_occur / total_transactions. use_colnames : bool (default: False) If true, uses the DataFrames' column names in the returned DataFrame instead of column indices. max_len : int (default: None) Given the set of all maximal itemsets, return those that are less than max_len . If None (default) all possible itemsets lengths are evaluated. verbose : int (default: 0) Shows the stages of conditional tree generation. Returns pandas DataFrame with columns ['support', 'itemsets'] of all maximal itemsets that are >= min_support and < than max_len (if max_len is not None). Each itemset in the 'itemsets' column is of type frozenset , which is a Python built-in type that behaves similarly to sets except that it is immutable (For more info, see https://docs.python.org/3.6/library/stdtypes.html#frozenset). Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/frequent_patterns/fpmax/","title":"Fpmax"},{"location":"user_guide/frequent_patterns/fpmax/#fpmax-maximal-itemsets-via-the-fp-max-algorithm","text":"Function implementing FP-Max to extract maximal itemsets for association rule mining from mlxtend.frequent_patterns import fpmax","title":"fpmax: Maximal itemsets via the FP-Max algorithm"},{"location":"user_guide/frequent_patterns/fpmax/#overview","text":"The Apriori algorithm is among the first and most popular algorithms for frequent itemset generation (frequent itemsets are then used for association rule mining). However, the runtime of Apriori can be quite large, especially for datasets with a large number of unique items, as the runtime grows exponentially depending on the number of unique items. In contrast to Apriori, FP-Growth is a frequent pattern generation algorithm that inserts items into a pattern search tree, which allows it to have a linear increase in runtime with respect to the number of unique items or entries. FP-Max is a variant of FP-Growth, which focuses on obtaining maximal itemsets. An itemset X is said to maximal if X is frequent and there exists no frequent super-pattern containing X. In other words, a frequent pattern X cannot be sub-pattern of larger frequent pattern to qualify for the definition maximal itemset .","title":"Overview"},{"location":"user_guide/frequent_patterns/fpmax/#references","text":"[1] Grahne, G., & Zhu, J. (2003, November). Efficiently using prefix-trees in mining frequent itemsets. In FIMI (Vol. 90).","title":"References"},{"location":"user_guide/frequent_patterns/fpmax/#related","text":"FP-Growth Apriori","title":"Related"},{"location":"user_guide/frequent_patterns/fpmax/#example-1-maximal-itemsets","text":"The fpmax function expects data in a one-hot encoded pandas DataFrame. Suppose we have the following transaction data: dataset = [['Milk', 'Onion', 'Nutmeg', 'Kidney Beans', 'Eggs', 'Yogurt'], ['Dill', 'Onion', 'Nutmeg', 'Kidney Beans', 'Eggs', 'Yogurt'], ['Milk', 'Apple', 'Kidney Beans', 'Eggs'], ['Milk', 'Unicorn', 'Corn', 'Kidney Beans', 'Yogurt'], ['Corn', 'Onion', 'Onion', 'Kidney Beans', 'Ice cream', 'Eggs']] We can transform it into the right format via the TransactionEncoder as follows: import pandas as pd from mlxtend.preprocessing import TransactionEncoder te = TransactionEncoder() te_ary = te.fit(dataset).transform(dataset) df = pd.DataFrame(te_ary, columns=te.columns_) df .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } Apple Corn Dill Eggs Ice cream Kidney Beans Milk Nutmeg Onion Unicorn Yogurt 0 False False False True False True True True True False True 1 False False True True False True False True True False True 2 True False False True False True True False False False False 3 False True False False False True True False False True True 4 False True False True True True False False True False False Now, let us return the items and itemsets with at least 60% support: from mlxtend.frequent_patterns import fpmax fpmax(df, min_support=0.6) .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } support itemsets 0 0.6 (5, 6) 1 0.6 (8, 3, 5) 2 0.6 (10, 5) By default, fpmax returns the column indices of the items, which may be useful in downstream operations such as association rule mining. For better readability, we can set use_colnames=True to convert these integer values into the respective item names: fpmax(df, min_support=0.6, use_colnames=True) .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } support itemsets 0 0.6 (Kidney Beans, Milk) 1 0.6 (Onion, Eggs, Kidney Beans) 2 0.6 (Kidney Beans, Yogurt)","title":"Example 1 -- Maximal Itemsets"},{"location":"user_guide/frequent_patterns/fpmax/#more-examples","text":"Please note that since the fpmax function is a drop-in replacement for fpgrowth and apriori , it comes with the same set of function arguments and return arguments. Thus, for more examples, please see the apriori documentation.","title":"More Examples"},{"location":"user_guide/frequent_patterns/fpmax/#api","text":"fpmax(df, min_support=0.5, use_colnames=False, max_len=None, verbose=0) Get maximal frequent itemsets from a one-hot DataFrame Parameters df : pandas DataFrame pandas DataFrame the encoded format. Also supports DataFrames with sparse data; for more info, please see (https://pandas.pydata.org/pandas-docs/stable/ user_guide/sparse.html#sparse-data-structures) Please note that the old pandas SparseDataFrame format is no longer supported in mlxtend >= 0.17.2. The allowed values are either 0/1 or True/False. For example, Apple Bananas Beer Chicken Milk Rice 0 True False True True False True 1 True False True False False True 2 True False True False False False 3 True True False False False False 4 False False True True True True 5 False False True False True True 6 False False True False True False 7 True True False False False False min_support : float (default: 0.5) A float between 0 and 1 for minimum support of the itemsets returned. The support is computed as the fraction transactions_where_item(s)_occur / total_transactions. use_colnames : bool (default: False) If true, uses the DataFrames' column names in the returned DataFrame instead of column indices. max_len : int (default: None) Given the set of all maximal itemsets, return those that are less than max_len . If None (default) all possible itemsets lengths are evaluated. verbose : int (default: 0) Shows the stages of conditional tree generation. Returns pandas DataFrame with columns ['support', 'itemsets'] of all maximal itemsets that are >= min_support and < than max_len (if max_len is not None). Each itemset in the 'itemsets' column is of type frozenset , which is a Python built-in type that behaves similarly to sets except that it is immutable (For more info, see https://docs.python.org/3.6/library/stdtypes.html#frozenset). Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/frequent_patterns/fpmax/","title":"API"},{"location":"user_guide/frequent_patterns/hmine/","text":"hmine : Frequent itemsets via the H-mine algorithm Hmine function to extract frequent itemsets for association rule mining from mlxtend.frequent_patterns import hmine Overview H-mine [1] (memory-based hyperstructure mining of frequent patterns) is a data mining algorithm used for frequent itemset mining -- the process of finding frequently occurring patterns in large transactional datasets. H-mine is an improvement over the Apriori and FP-Growth algorithms, offering better performance in terms of time and space complexity. It achieves this by using the H-struct data structure and a more efficient search space traversal method. H-mine improves upon the FP-Growth algorithm by introducing a novel data structure called the H-struct. The H-struct is a hybrid data structure that combines the benefits of both horizontal and vertical data layouts, making it more efficient for frequent itemset mining. A distinct feature of this method is that it has very limited and precisely predictable space overhead and runs really fast in memory-based settings. Moreover, it can be scaled up to very large databases by database partitioning, and when the data set becomes dense, (conditional) FP-trees can be constructed dynamically as part of the mining process. References [1] Pei J, Han J, Lu H, Nishio S, Tang S and Yang D, \" H-Mine: Fast and space-preserving frequent pattern mining in large databases. \" IIE Transactions, Vol. 39, pp. 593\u2013605, 2007. Related FP-Growth FP-Max Apriori Example 1 -- Generating Frequent Itemsets The hmine function expects data in a one-hot encoded pandas DataFrame. Suppose we have the following transaction data: dataset = [['Milk', 'Onion', 'Nutmeg', 'Kidney Beans', 'Eggs', 'Yogurt'], ['Dill', 'Onion', 'Nutmeg', 'Kidney Beans', 'Eggs', 'Yogurt'], ['Milk', 'Apple', 'Kidney Beans', 'Eggs'], ['Milk', 'Unicorn', 'Corn', 'Kidney Beans', 'Yogurt'], ['Corn', 'Onion', 'Onion', 'Kidney Beans', 'Ice cream', 'Eggs']] We can transform it into the right format via the TransactionEncoder as follows: import pandas as pd from mlxtend.preprocessing import TransactionEncoder te = TransactionEncoder() te_ary = te.fit(dataset).transform(dataset) df = pd.DataFrame(te_ary, columns=te.columns_) df .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } Apple Corn Dill Eggs Ice cream Kidney Beans Milk Nutmeg Onion Unicorn Yogurt 0 False False False True False True True True True False True 1 False False True True False True False True True False True 2 True False False True False True True False False False False 3 False True False False False True True False False True True 4 False True False True True True False False True False False Now, let us return the items and itemsets with at least 60% support: from mlxtend.frequent_patterns import hmine hmine(df, min_support=0.6) .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } support itemsets 0 0.8 (3) 1 0.8 (3, 5) 2 0.6 (8, 3, 5) 3 0.6 (8, 3) 4 1.0 (5) 5 0.6 (5, 6) 6 0.6 (8, 5) 7 0.6 (10, 5) 8 0.6 (6) 9 0.6 (8) 10 0.6 (10) By default, hmine returns the column indices of the items, which may be useful in downstream operations such as association rule mining. For better readability, we can set use_colnames=True to convert these integer values into the respective item names: hmine(df, min_support=0.6, use_colnames=True) .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } support itemsets 0 0.8 (Eggs) 1 0.8 (Eggs, Kidney Beans) 2 0.6 (Eggs, Kidney Beans, Onion) 3 0.6 (Eggs, Onion) 4 1.0 (Kidney Beans) 5 0.6 (Milk, Kidney Beans) 6 0.6 (Kidney Beans, Onion) 7 0.6 (Yogurt, Kidney Beans) 8 0.6 (Milk) 9 0.6 (Onion) 10 0.6 (Yogurt) Example 2 -- H-Mine versus Apriori and FP-Growth Since the hmine algorithm is a memory-based algorithm, it can be magnitudes faster than the alternative Apriori algorithm for large datasets. However, it can be much slower than the FP-Growth algorithm. In the following example, we compare the performance of hmine with the apriori and fpgrowth algorithms on a small dataset. import pandas as pd from mlxtend.preprocessing import TransactionEncoder te = TransactionEncoder() te_ary = te.fit(dataset).transform(dataset) df = pd.DataFrame(te_ary, columns=te.columns_) from mlxtend.frequent_patterns import apriori %timeit -n 100 -r 10 apriori(df, min_support=0.6, use_colnames=True) 3.41 ms \u00b1 584 \u00b5s per loop (mean \u00b1 std. dev. of 10 runs, 100 loops each) %timeit -n 100 -r 10 apriori(df, min_support=0.6, use_colnames=True, low_memory=True) 3.36 ms \u00b1 404 \u00b5s per loop (mean \u00b1 std. dev. of 10 runs, 100 loops each) from mlxtend.frequent_patterns import fpgrowth %timeit -n 100 -r 10 fpgrowth(df, min_support=0.6, use_colnames=True) 1.18 ms \u00b1 76.7 \u00b5s per loop (mean \u00b1 std. dev. of 10 runs, 100 loops each) from mlxtend.frequent_patterns import hmine %timeit -n 100 -r 10 hmine(df, min_support=0.6, use_colnames=True) 1.44 ms \u00b1 94.8 \u00b5s per loop (mean \u00b1 std. dev. of 10 runs, 100 loops each) Example 3 -- Working with Sparse Representations To save memory, you may want to represent your transaction data in the sparse format. This is especially useful if you have lots of products and small transactions. oht_ary = te.fit(dataset).transform(dataset, sparse=True) sparse_df = pd.DataFrame.sparse.from_spmatrix(oht_ary, columns=te.columns_) sparse_df .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } Apple Corn Dill Eggs Ice cream Kidney Beans Milk Nutmeg Onion Unicorn Yogurt 0 0 0 0 1 0 True 1 1 1 0 1 1 0 0 1 1 0 True 0 1 1 0 1 2 1 0 0 1 0 True 1 0 0 0 0 3 0 1 0 0 0 True 1 0 0 1 1 4 0 1 0 1 1 True 0 0 1 0 0 hmine(sparse_df, min_support=0.6, use_colnames=True, verbose=1) 2 itemset(s) from the suffixes on item(s) (Eggs) 1 itemset(s) from the suffixes on item(s) (Eggs, Kidney Beans) 0 itemset(s) from the suffixes on item(s) (Eggs, Kidney Beans, Onion) 0 itemset(s) from the suffixes on item(s) (Eggs, Onion) 3 itemset(s) from the suffixes on item(s) (Kidney Beans) 0 itemset(s) from the suffixes on item(s) (Kidney Beans, Milk) 0 itemset(s) from the suffixes on item(s) (Kidney Beans, Onion) 0 itemset(s) from the suffixes on item(s) (Kidney Beans, Yogurt) 0 itemset(s) from the suffixes on item(s) (Milk) 0 itemset(s) from the suffixes on item(s) (Onion) 0 itemset(s) from the suffixes on item(s) (Yogurt) .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } support itemsets 0 0.8 (Eggs) 1 0.8 (Eggs, Kidney Beans) 2 0.6 (Eggs, Kidney Beans, Onion) 3 0.6 (Eggs, Onion) 4 1.0 (Kidney Beans) 5 0.6 (Milk, Kidney Beans) 6 0.6 (Kidney Beans, Onion) 7 0.6 (Yogurt, Kidney Beans) 8 0.6 (Milk) 9 0.6 (Onion) 10 0.6 (Yogurt) More Examples Please note that since the hmine function is a drop-in replacement for apriori and fpgrowth , it comes with the same set of function arguments and return arguments. Thus, for more examples, please see the apriori documentation. API hmine(df, min_support=0.5, use_colnames=False, max_len=None, verbose=0) -> pandas.core.frame.DataFrame Get frequent itemsets from a one-hot DataFrame Parameters df : pandas DataFrame pandas DataFrame the encoded format. Also supports DataFrames with sparse data; for more info, please see https://pandas.pydata.org/pandas-docs/stable/user_guide/sparse.html#sparse-data-structures. Please note that the old pandas SparseDataFrame format is no longer supported in mlxtend >= 0.17.2. The allowed values are either 0/1 or True/False. For example, Apple Bananas Beer Chicken Milk Rice 0 True False True True False True 1 True False True False False True 2 True False True False False False 3 True True False False False False 4 False False True True True True 5 False False True False True True 6 False False True False True False 7 True True False False False False min_support : float (default: 0.5) A float between 0 and 1 for minimum support of the itemsets returned. The support is computed as the fraction transactions_where_item(s)_occur / total_transactions. use_colnames : bool (default: False) If true, uses the DataFrames' column names in the returned DataFrame instead of column indices. max_len : int (default: None) Maximum length of the itemsets generated. If None (default) all possible itemsets lengths are evaluated. verbose : int (default: 0) Shows the stages of conditional tree generation. Returns pandas DataFrame with columns ['support', 'itemsets'] of all itemsets that are >= min_support and < than max_len (if max_len is not None). Each itemset in the 'itemsets' column is of type frozenset , which is a Python built-in type that behaves similarly to sets except that it is immutable (For more info, see https://docs.python.org/3.6/library/stdtypes.html#frozenset). Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/frequent_patterns/hmine/ ython","title":"Hmine"},{"location":"user_guide/frequent_patterns/hmine/#hmine-frequent-itemsets-via-the-h-mine-algorithm","text":"Hmine function to extract frequent itemsets for association rule mining from mlxtend.frequent_patterns import hmine","title":"hmine: Frequent itemsets via the H-mine algorithm"},{"location":"user_guide/frequent_patterns/hmine/#overview","text":"H-mine [1] (memory-based hyperstructure mining of frequent patterns) is a data mining algorithm used for frequent itemset mining -- the process of finding frequently occurring patterns in large transactional datasets. H-mine is an improvement over the Apriori and FP-Growth algorithms, offering better performance in terms of time and space complexity. It achieves this by using the H-struct data structure and a more efficient search space traversal method. H-mine improves upon the FP-Growth algorithm by introducing a novel data structure called the H-struct. The H-struct is a hybrid data structure that combines the benefits of both horizontal and vertical data layouts, making it more efficient for frequent itemset mining. A distinct feature of this method is that it has very limited and precisely predictable space overhead and runs really fast in memory-based settings. Moreover, it can be scaled up to very large databases by database partitioning, and when the data set becomes dense, (conditional) FP-trees can be constructed dynamically as part of the mining process.","title":"Overview"},{"location":"user_guide/frequent_patterns/hmine/#references","text":"[1] Pei J, Han J, Lu H, Nishio S, Tang S and Yang D, \" H-Mine: Fast and space-preserving frequent pattern mining in large databases. \" IIE Transactions, Vol. 39, pp. 593\u2013605, 2007.","title":"References"},{"location":"user_guide/frequent_patterns/hmine/#related","text":"FP-Growth FP-Max Apriori","title":"Related"},{"location":"user_guide/frequent_patterns/hmine/#example-1-generating-frequent-itemsets","text":"The hmine function expects data in a one-hot encoded pandas DataFrame. Suppose we have the following transaction data: dataset = [['Milk', 'Onion', 'Nutmeg', 'Kidney Beans', 'Eggs', 'Yogurt'], ['Dill', 'Onion', 'Nutmeg', 'Kidney Beans', 'Eggs', 'Yogurt'], ['Milk', 'Apple', 'Kidney Beans', 'Eggs'], ['Milk', 'Unicorn', 'Corn', 'Kidney Beans', 'Yogurt'], ['Corn', 'Onion', 'Onion', 'Kidney Beans', 'Ice cream', 'Eggs']] We can transform it into the right format via the TransactionEncoder as follows: import pandas as pd from mlxtend.preprocessing import TransactionEncoder te = TransactionEncoder() te_ary = te.fit(dataset).transform(dataset) df = pd.DataFrame(te_ary, columns=te.columns_) df .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } Apple Corn Dill Eggs Ice cream Kidney Beans Milk Nutmeg Onion Unicorn Yogurt 0 False False False True False True True True True False True 1 False False True True False True False True True False True 2 True False False True False True True False False False False 3 False True False False False True True False False True True 4 False True False True True True False False True False False Now, let us return the items and itemsets with at least 60% support: from mlxtend.frequent_patterns import hmine hmine(df, min_support=0.6) .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } support itemsets 0 0.8 (3) 1 0.8 (3, 5) 2 0.6 (8, 3, 5) 3 0.6 (8, 3) 4 1.0 (5) 5 0.6 (5, 6) 6 0.6 (8, 5) 7 0.6 (10, 5) 8 0.6 (6) 9 0.6 (8) 10 0.6 (10) By default, hmine returns the column indices of the items, which may be useful in downstream operations such as association rule mining. For better readability, we can set use_colnames=True to convert these integer values into the respective item names: hmine(df, min_support=0.6, use_colnames=True) .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } support itemsets 0 0.8 (Eggs) 1 0.8 (Eggs, Kidney Beans) 2 0.6 (Eggs, Kidney Beans, Onion) 3 0.6 (Eggs, Onion) 4 1.0 (Kidney Beans) 5 0.6 (Milk, Kidney Beans) 6 0.6 (Kidney Beans, Onion) 7 0.6 (Yogurt, Kidney Beans) 8 0.6 (Milk) 9 0.6 (Onion) 10 0.6 (Yogurt)","title":"Example 1 -- Generating Frequent Itemsets"},{"location":"user_guide/frequent_patterns/hmine/#example-2-h-mine-versus-apriori-and-fp-growth","text":"Since the hmine algorithm is a memory-based algorithm, it can be magnitudes faster than the alternative Apriori algorithm for large datasets. However, it can be much slower than the FP-Growth algorithm. In the following example, we compare the performance of hmine with the apriori and fpgrowth algorithms on a small dataset. import pandas as pd from mlxtend.preprocessing import TransactionEncoder te = TransactionEncoder() te_ary = te.fit(dataset).transform(dataset) df = pd.DataFrame(te_ary, columns=te.columns_) from mlxtend.frequent_patterns import apriori %timeit -n 100 -r 10 apriori(df, min_support=0.6, use_colnames=True) 3.41 ms \u00b1 584 \u00b5s per loop (mean \u00b1 std. dev. of 10 runs, 100 loops each) %timeit -n 100 -r 10 apriori(df, min_support=0.6, use_colnames=True, low_memory=True) 3.36 ms \u00b1 404 \u00b5s per loop (mean \u00b1 std. dev. of 10 runs, 100 loops each) from mlxtend.frequent_patterns import fpgrowth %timeit -n 100 -r 10 fpgrowth(df, min_support=0.6, use_colnames=True) 1.18 ms \u00b1 76.7 \u00b5s per loop (mean \u00b1 std. dev. of 10 runs, 100 loops each) from mlxtend.frequent_patterns import hmine %timeit -n 100 -r 10 hmine(df, min_support=0.6, use_colnames=True) 1.44 ms \u00b1 94.8 \u00b5s per loop (mean \u00b1 std. dev. of 10 runs, 100 loops each)","title":"Example 2 -- H-Mine versus Apriori and FP-Growth"},{"location":"user_guide/frequent_patterns/hmine/#example-3-working-with-sparse-representations","text":"To save memory, you may want to represent your transaction data in the sparse format. This is especially useful if you have lots of products and small transactions. oht_ary = te.fit(dataset).transform(dataset, sparse=True) sparse_df = pd.DataFrame.sparse.from_spmatrix(oht_ary, columns=te.columns_) sparse_df .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } Apple Corn Dill Eggs Ice cream Kidney Beans Milk Nutmeg Onion Unicorn Yogurt 0 0 0 0 1 0 True 1 1 1 0 1 1 0 0 1 1 0 True 0 1 1 0 1 2 1 0 0 1 0 True 1 0 0 0 0 3 0 1 0 0 0 True 1 0 0 1 1 4 0 1 0 1 1 True 0 0 1 0 0 hmine(sparse_df, min_support=0.6, use_colnames=True, verbose=1) 2 itemset(s) from the suffixes on item(s) (Eggs) 1 itemset(s) from the suffixes on item(s) (Eggs, Kidney Beans) 0 itemset(s) from the suffixes on item(s) (Eggs, Kidney Beans, Onion) 0 itemset(s) from the suffixes on item(s) (Eggs, Onion) 3 itemset(s) from the suffixes on item(s) (Kidney Beans) 0 itemset(s) from the suffixes on item(s) (Kidney Beans, Milk) 0 itemset(s) from the suffixes on item(s) (Kidney Beans, Onion) 0 itemset(s) from the suffixes on item(s) (Kidney Beans, Yogurt) 0 itemset(s) from the suffixes on item(s) (Milk) 0 itemset(s) from the suffixes on item(s) (Onion) 0 itemset(s) from the suffixes on item(s) (Yogurt) .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } support itemsets 0 0.8 (Eggs) 1 0.8 (Eggs, Kidney Beans) 2 0.6 (Eggs, Kidney Beans, Onion) 3 0.6 (Eggs, Onion) 4 1.0 (Kidney Beans) 5 0.6 (Milk, Kidney Beans) 6 0.6 (Kidney Beans, Onion) 7 0.6 (Yogurt, Kidney Beans) 8 0.6 (Milk) 9 0.6 (Onion) 10 0.6 (Yogurt)","title":"Example 3 -- Working with Sparse Representations"},{"location":"user_guide/frequent_patterns/hmine/#more-examples","text":"Please note that since the hmine function is a drop-in replacement for apriori and fpgrowth , it comes with the same set of function arguments and return arguments. Thus, for more examples, please see the apriori documentation.","title":"More Examples"},{"location":"user_guide/frequent_patterns/hmine/#api","text":"hmine(df, min_support=0.5, use_colnames=False, max_len=None, verbose=0) -> pandas.core.frame.DataFrame Get frequent itemsets from a one-hot DataFrame Parameters df : pandas DataFrame pandas DataFrame the encoded format. Also supports DataFrames with sparse data; for more info, please see https://pandas.pydata.org/pandas-docs/stable/user_guide/sparse.html#sparse-data-structures. Please note that the old pandas SparseDataFrame format is no longer supported in mlxtend >= 0.17.2. The allowed values are either 0/1 or True/False. For example, Apple Bananas Beer Chicken Milk Rice 0 True False True True False True 1 True False True False False True 2 True False True False False False 3 True True False False False False 4 False False True True True True 5 False False True False True True 6 False False True False True False 7 True True False False False False min_support : float (default: 0.5) A float between 0 and 1 for minimum support of the itemsets returned. The support is computed as the fraction transactions_where_item(s)_occur / total_transactions. use_colnames : bool (default: False) If true, uses the DataFrames' column names in the returned DataFrame instead of column indices. max_len : int (default: None) Maximum length of the itemsets generated. If None (default) all possible itemsets lengths are evaluated. verbose : int (default: 0) Shows the stages of conditional tree generation. Returns pandas DataFrame with columns ['support', 'itemsets'] of all itemsets that are >= min_support and < than max_len (if max_len is not None). Each itemset in the 'itemsets' column is of type frozenset , which is a Python built-in type that behaves similarly to sets except that it is immutable (For more info, see https://docs.python.org/3.6/library/stdtypes.html#frozenset). Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/frequent_patterns/hmine/ ython","title":"API"},{"location":"user_guide/image/extract_face_landmarks/","text":"extract_face_landmarks: extract 68 landmark features from face images A function extract facial landmarks. from mlxtend.image import extract_face_landmarks Overview The extract_face_landmarks function detects the faces in a given image, and then it will return the face landmark points (also known as face shape) for the first found face in the image based on dlib's face landmark detection code (https://dlib.net/face_landmark_detection_ex.cpp.html): The face detector we use is made using the classic Histogram of Oriented Gradients (HOG) feature combined with a linear classifier, an image pyramid, and sliding window detection scheme. The pose estimator was created by using dlib's implementation of the paper: One Millisecond Face Alignment with an Ensemble of Regression Trees by Vahid Kazemi and Josephine Sullivan, CVPR 2014 and was trained on the iBUG 300-W face landmark dataset (see https://ibug.doc.ic.ac.uk/resources/facial-point-annotations/): C. Sagonas, E. Antonakos, G, Tzimiropoulos, S. Zafeiriou, M. Pantic. 300 faces In-the-wild challenge: Database and results. Image and Vision Computing (IMAVIS), Special Issue on Facial Landmark Localisation \"In-The-Wild\". 2016. You can get the trained model file from: https://dlib.net/files/shape_predictor_68_face_landmarks.dat.bz2. Note that the license for the iBUG 300-W dataset excludes commercial use. So you should contact Imperial College London to find out if it's OK for you to use this model file in a commercial product. References Kazemi, Vahid, and Josephine Sullivan. \"One millisecond face alignment with an ensemble of regression trees.\" Proceedings of the IEEE Conference on Computer Vision and Pattern Recognition. 2014. Example 1 -- RGB Images import imageio import matplotlib.pyplot as plt from mlxtend.image import extract_face_landmarks img = imageio.imread('test-face.png') landmarks = extract_face_landmarks(img) print(landmarks.shape) print('\\n\\nFirst 10 landmarks:\\n', landmarks[:10]) (68, 2) First 10 landmarks: [[132 159] [134 173] [137 187] [142 200] [150 212] [159 222] [170 231] [184 235] [198 236] [211 231]] Visualization of the landmarks: fig = plt.figure(figsize=(15, 5)) ax = fig.add_subplot(1, 3, 1) ax.imshow(img) ax = fig.add_subplot(1, 3, 2) ax.scatter(landmarks[:, 0], -landmarks[:, 1], alpha=0.8) ax = fig.add_subplot(1, 3, 3) img2 = img.copy() for p in landmarks: img2[p[1]-3:p[1]+3, p[0]-3:p[0]+3, :] = (255, 255, 255) # note that the values -3 and +3 will make the landmarks # overlayed on the image 6 pixels wide; depending on the # resolution of the face image, you may want to change # this value ax.imshow(img2) plt.show() Displaying the index of landmark points import numpy as np left_eye = np.array([36, 37, 38, 39, 40, 41]) right_eye = np.array([42, 43, 44, 45, 46, 47]) import matplotlib.pyplot as plt %matplotlib inline fig = plt.figure(figsize=(10,10)) plt.plot(landmarks[:,0], -landmarks[:,1], 'ro', markersize=8, alpha = 0.5) for i in range(landmarks.shape[0]): plt.text(landmarks[i,0]+1, -landmarks[i,1], str(i), size=14) left_eye_center = np.mean(landmarks[left_eye], axis=0) right_eye_center = np.mean(landmarks[right_eye], axis=0) print('Coordinates of the Left Eye: ', left_eye_center) print('Coordinates of the Right Eye: ', right_eye_center) plt.plot([left_eye_center[0]], [-left_eye_center[1]], marker='+', color='blue', markersize=10, mew=4) plt.plot([right_eye_center[0]], [-right_eye_center[1]], marker='+', color='blue', markersize=10, mew=4) plt.xticks([]) plt.yticks([]) plt.show() Coordinates of the Left Eye: [169.33333333 156. ] Coordinates of the Right Eye: [210.83333333 152.16666667] Example 2 -- Grayscale Images import imageio import matplotlib.pyplot as plt from mlxtend.image import extract_face_landmarks img = imageio.imread('test-face_grayscale_lowres.png', ) landmarks = extract_face_landmarks(img) print(landmarks.shape) print('\\n\\nFirst 10 landmarks:\\n', landmarks[:10]) (68, 2) First 10 landmarks: [[37 42] [38 46] [39 51] [41 55] [44 59] [47 62] [51 65] [55 66] [60 66] [64 64]] fig = plt.figure(figsize=(15, 5)) ax = fig.add_subplot(1, 3, 1) ax.imshow(img, cmap='gray') ax = fig.add_subplot(1, 3, 2) ax.scatter(landmarks[:, 0], -landmarks[:, 1], alpha=0.8) ax = fig.add_subplot(1, 3, 3) img2 = img.copy() for p in landmarks: img2[p[1]-1:p[1]+1, p[0]-1:p[0]+1] = 255 # note that the values -1 and +1 will make the landmarks # overlayed on the image 2 pixels wide; depending on the # resolution of the face image, you may want to change # this value ax.imshow(img2, cmap='gray') plt.show() API extract_face_landmarks(img, return_dtype= ) Function to extract face landmarks. Note that this function requires an installation of the Python version of the library \"dlib\": https://dlib.net Parameters img : array, shape = [h, w, ?] Numpy array of a face image or imageio.core.util.Array. E.g., img = imageio.core.util.Array(ary) Supported shapes are - 3D tensors with 1 or more color channels, for example, RGB: [h, w, 3] - 2D tensors without color channel, for example, Grayscale: [h, w] return_dtype: the return data-type of the array, default: np.int32. Returns landmarks : numpy.ndarray, shape = [68, 2] A numpy array, where each row contains a landmark/point x-y coordinates. Return None if no face is detected by Dlib. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/image/extract_face_landmarks/","title":"extract_face_landmarks: extract 68 landmark features from face images"},{"location":"user_guide/image/extract_face_landmarks/#extract_face_landmarks-extract-68-landmark-features-from-face-images","text":"A function extract facial landmarks. from mlxtend.image import extract_face_landmarks","title":"extract_face_landmarks: extract 68 landmark features from face images"},{"location":"user_guide/image/extract_face_landmarks/#overview","text":"The extract_face_landmarks function detects the faces in a given image, and then it will return the face landmark points (also known as face shape) for the first found face in the image based on dlib's face landmark detection code (https://dlib.net/face_landmark_detection_ex.cpp.html): The face detector we use is made using the classic Histogram of Oriented Gradients (HOG) feature combined with a linear classifier, an image pyramid, and sliding window detection scheme. The pose estimator was created by using dlib's implementation of the paper: One Millisecond Face Alignment with an Ensemble of Regression Trees by Vahid Kazemi and Josephine Sullivan, CVPR 2014 and was trained on the iBUG 300-W face landmark dataset (see https://ibug.doc.ic.ac.uk/resources/facial-point-annotations/): C. Sagonas, E. Antonakos, G, Tzimiropoulos, S. Zafeiriou, M. Pantic. 300 faces In-the-wild challenge: Database and results. Image and Vision Computing (IMAVIS), Special Issue on Facial Landmark Localisation \"In-The-Wild\". 2016. You can get the trained model file from: https://dlib.net/files/shape_predictor_68_face_landmarks.dat.bz2. Note that the license for the iBUG 300-W dataset excludes commercial use. So you should contact Imperial College London to find out if it's OK for you to use this model file in a commercial product.","title":"Overview"},{"location":"user_guide/image/extract_face_landmarks/#references","text":"Kazemi, Vahid, and Josephine Sullivan. \"One millisecond face alignment with an ensemble of regression trees.\" Proceedings of the IEEE Conference on Computer Vision and Pattern Recognition. 2014.","title":"References"},{"location":"user_guide/image/extract_face_landmarks/#example-1-rgb-images","text":"import imageio import matplotlib.pyplot as plt from mlxtend.image import extract_face_landmarks img = imageio.imread('test-face.png') landmarks = extract_face_landmarks(img) print(landmarks.shape) print('\\n\\nFirst 10 landmarks:\\n', landmarks[:10]) (68, 2) First 10 landmarks: [[132 159] [134 173] [137 187] [142 200] [150 212] [159 222] [170 231] [184 235] [198 236] [211 231]] Visualization of the landmarks: fig = plt.figure(figsize=(15, 5)) ax = fig.add_subplot(1, 3, 1) ax.imshow(img) ax = fig.add_subplot(1, 3, 2) ax.scatter(landmarks[:, 0], -landmarks[:, 1], alpha=0.8) ax = fig.add_subplot(1, 3, 3) img2 = img.copy() for p in landmarks: img2[p[1]-3:p[1]+3, p[0]-3:p[0]+3, :] = (255, 255, 255) # note that the values -3 and +3 will make the landmarks # overlayed on the image 6 pixels wide; depending on the # resolution of the face image, you may want to change # this value ax.imshow(img2) plt.show()","title":"Example 1 -- RGB Images"},{"location":"user_guide/image/extract_face_landmarks/#displaying-the-index-of-landmark-points","text":"import numpy as np left_eye = np.array([36, 37, 38, 39, 40, 41]) right_eye = np.array([42, 43, 44, 45, 46, 47]) import matplotlib.pyplot as plt %matplotlib inline fig = plt.figure(figsize=(10,10)) plt.plot(landmarks[:,0], -landmarks[:,1], 'ro', markersize=8, alpha = 0.5) for i in range(landmarks.shape[0]): plt.text(landmarks[i,0]+1, -landmarks[i,1], str(i), size=14) left_eye_center = np.mean(landmarks[left_eye], axis=0) right_eye_center = np.mean(landmarks[right_eye], axis=0) print('Coordinates of the Left Eye: ', left_eye_center) print('Coordinates of the Right Eye: ', right_eye_center) plt.plot([left_eye_center[0]], [-left_eye_center[1]], marker='+', color='blue', markersize=10, mew=4) plt.plot([right_eye_center[0]], [-right_eye_center[1]], marker='+', color='blue', markersize=10, mew=4) plt.xticks([]) plt.yticks([]) plt.show() Coordinates of the Left Eye: [169.33333333 156. ] Coordinates of the Right Eye: [210.83333333 152.16666667]","title":"Displaying the index of landmark points"},{"location":"user_guide/image/extract_face_landmarks/#example-2-grayscale-images","text":"import imageio import matplotlib.pyplot as plt from mlxtend.image import extract_face_landmarks img = imageio.imread('test-face_grayscale_lowres.png', ) landmarks = extract_face_landmarks(img) print(landmarks.shape) print('\\n\\nFirst 10 landmarks:\\n', landmarks[:10]) (68, 2) First 10 landmarks: [[37 42] [38 46] [39 51] [41 55] [44 59] [47 62] [51 65] [55 66] [60 66] [64 64]] fig = plt.figure(figsize=(15, 5)) ax = fig.add_subplot(1, 3, 1) ax.imshow(img, cmap='gray') ax = fig.add_subplot(1, 3, 2) ax.scatter(landmarks[:, 0], -landmarks[:, 1], alpha=0.8) ax = fig.add_subplot(1, 3, 3) img2 = img.copy() for p in landmarks: img2[p[1]-1:p[1]+1, p[0]-1:p[0]+1] = 255 # note that the values -1 and +1 will make the landmarks # overlayed on the image 2 pixels wide; depending on the # resolution of the face image, you may want to change # this value ax.imshow(img2, cmap='gray') plt.show()","title":"Example 2 -- Grayscale Images"},{"location":"user_guide/image/extract_face_landmarks/#api","text":"extract_face_landmarks(img, return_dtype= ) Function to extract face landmarks. Note that this function requires an installation of the Python version of the library \"dlib\": https://dlib.net Parameters img : array, shape = [h, w, ?] Numpy array of a face image or imageio.core.util.Array. E.g., img = imageio.core.util.Array(ary) Supported shapes are - 3D tensors with 1 or more color channels, for example, RGB: [h, w, 3] - 2D tensors without color channel, for example, Grayscale: [h, w] return_dtype: the return data-type of the array, default: np.int32. Returns landmarks : numpy.ndarray, shape = [68, 2] A numpy array, where each row contains a landmark/point x-y coordinates. Return None if no face is detected by Dlib. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/image/extract_face_landmarks/","title":"API"},{"location":"user_guide/image/eyepad_align/","text":"EyepadAlign: align face images based on eye location A class to align face images based on eye location. from mlxtend.image import EyepadAlign Overview The EyepadAlign class align face images to target face landmarks based on the location of the eyes. The facial landmarks are detected using mlxted.image.extract_face_landmarks , which is based on dlib's face landmark detection code [1]. EyepadAlign implements 3 fit methods, fit_image , fit_values , and fit_directory ; once the EyepadAlign object has been fitted via one of these methods (as illustrated in the examples), it can be used to transform a target image. References [1] Kazemi, Vahid, and Josephine Sullivan. \"One millisecond face alignment with an ensemble of regression trees.\" Proceedings of the IEEE Conference on Computer Vision and Pattern Recognition. 2014. Example 1 (fit_image) This example illustrates the use of EyepadAlign fit to a single target input image (first image in the CelebA dataset ) to another portrait such that the eye position is in the same scale and location as the target image. import imageio from mlxtend.image import EyepadAlign ### Fit EyepadAlign on a single target image eyepad = EyepadAlign() target_image = imageio.imread('celeba-subset/000001.jpg') print('Target image shape: ', target_image.shape) eyepad.fit_image(target_image); Target image shape: (218, 178, 3) ### Transform a test image img = imageio.imread('test-face.png') print('Original Image shape: ', img.shape) img_tr = eyepad.transform(img) print('Transformed Image shape: ', img_tr.shape) Original Image shape: (350, 350, 3) Transformed Image shape: (218, 178, 3) import matplotlib.pyplot as plt fig = plt.figure(figsize=(12, 5)) ax = fig.add_subplot(1, 3, 1) ax.imshow(target_image) ax.set_title('Target', size=20) ax = fig.add_subplot(1, 3, 2) ax.imshow(img) ax.set_title('Original', size=20) ax = fig.add_subplot(1, 3, 3) ax.imshow(img_tr) ax.set_title('Aligned to Target', size=20) plt.show() Example 2 (fit_directory) In typical computer vision and deep learning tasks, we are interested in aligning all images in a dataset and across dataset to allow cross-dataset comparisons of face image classifiers. Using the fit_directory method, we can fit EyepadAlign such that the average landmarks are computed from all images in a directory. Then, we can use these average landmarks to transform new images based on the average eye location. import imageio from mlxtend.image import EyepadAlign ### Setting verbosity to 1 will print a progress bar eyepad = EyepadAlign(verbose=1) eyepad.fit_directory(target_img_dir='celeba-subset/', target_width=178, target_height=218, file_extension='.jpg') img = imageio.imread('test-face.png') img_tr = eyepad.transform(img) print('Transformed Image shape: ', img_tr.shape) Pre-Checking directory for consistent image dimensions... 0% [#########] 100% | ETA: 00:00:00 Fitting the average facial landmarks for 9 face images Total time elapsed: 00:00:00 0% [##### ] 100% | ETA: 00:00:00/Users/sebastian/code/mlxtend/mlxtend/image/extract_face_landmarks.py:66: UserWarning: No face detected. warnings.warn('No face detected.') /Users/sebastian/code/mlxtend/mlxtend/image/eyepad_align.py:188: UserWarning: No face detected in image 000004.jpg. Image ignored. % f) 0% [#########] 100% | ETA: 00:00:00 Total time elapsed: 00:00:00 /Users/sebastian/code/mlxtend/mlxtend/image/eyepad_align.py:188: UserWarning: No face detected in image 000003.jpg. Image ignored. % f) Transformed Image shape: (218, 178, 3) Note that a warning is displayed if no face landmarks could be detected. In these cases, the image is skipped and not included in the average landmark computation. E.g., 000003.jpg would be such an example: import matplotlib.pyplot as plt img_no_landmarks = imageio.imread('celeba-subset/000003.jpg') plt.imshow(img_no_landmarks) plt.show() Finally, the an example of aligning the original face image to the average landmarks in the dataset is shown below: fig = plt.figure(figsize=(10, 4)) ax = fig.add_subplot(1, 3, 1) ax.scatter(eyepad.target_landmarks_[:, 0], eyepad.target_landmarks_[:, 1], alpha=0.8) ax.set_title('Target Landmarks', size=20) ax.set_xlim(0, 178) ax.set_ylim(218, 0) ax = fig.add_subplot(1, 3, 2) ax.imshow(img) ax.set_title('Original', size=20) ax = fig.add_subplot(1, 3, 3) ax.imshow(img_tr) ax.set_title('Aligned to Target', size=20) plt.tight_layout() plt.show() Example 3 (fit_values) For more customization, landmarks can also be passed manually. import imageio from mlxtend.image import EyepadAlign import numpy as np celeba_landmarks = np.array([[45, 113], [45, 124], [46, 136], [49, 147], [53, 158], [60, 167], [68, 175], [77, 182], [88, 184], [99, 182], [108, 175], [117, 167], [124, 158], [128, 147], [131, 136], [132, 124], [132, 113], [53, 104], [58, 99], [66, 98], [73, 99], [80, 102], [95, 102], [103, 99], [110, 98], [118, 99], [123, 104], [88, 111], [88, 119], [88, 126], [88, 134], [79, 139], [83, 140], [88, 142], [92, 140], [96, 139], [62, 112], [66, 109], [72, 109], [77, 113], [71, 114], [66, 114], [99, 113], [104, 109], [110, 109], [115, 112], [110, 114], [105, 114], [71, 152], [77, 150], [83, 149], [88, 150], [92, 149], [99, 150], [105, 152], [99, 159], [93, 162], [88, 163], [83, 162], [77, 159], [73, 153], [83, 152], [88, 153], [93, 152], [103, 153], [93, 156], [88, 157], [83, 156]]) eyepad = EyepadAlign(verbose=1) eyepad.fit_values(target_landmarks=celeba_landmarks, target_width=178, target_height=218); import matplotlib.pyplot as plt img = imageio.imread('test-face.png') img_tr = eyepad.transform(img) print('Transformed Image shape: ', img_tr.shape) fig = plt.figure(figsize=(10, 4)) ax = fig.add_subplot(1, 3, 1) ax.scatter(eyepad.target_landmarks_[:, 0], eyepad.target_landmarks_[:, 1], alpha=0.8) ax.set_title('Target Landmarks', size=20) ax.set_xlim(0, 178) ax.set_ylim(218, 0) ax = fig.add_subplot(1, 3, 2) ax.imshow(img) ax.set_title('Original', size=20) ax = fig.add_subplot(1, 3, 3) ax.imshow(img_tr) ax.set_title('Aligned to Target', size=20) plt.tight_layout() plt.show() Transformed Image shape: (218, 178, 3) Example 4 (centering the nose) In many applications that involve center-cropping, it is helpful to translate the image such that the tip of the nose is located in the center of the image. ### Fit EyepadAlign to dataset to compute average landmarks import imageio from mlxtend.image import EyepadAlign import numpy as np eyepad = EyepadAlign(verbose=1) eyepad.fit_directory(target_img_dir='celeba-subset/', target_width=178, target_height=218, file_extension='.jpg') Pre-Checking directory for consistent image dimensions... 0% [#########] 100% | ETA: 00:00:00 Fitting the average facial landmarks for 9 face images Total time elapsed: 00:00:00 0% [#########] 100% | ETA: 00:00:00 Total time elapsed: 00:00:00 <mlxtend.image.eyepad_align.EyepadAlign at 0x7fa09865d390> ### Use the calculated landmarks to center the nose: center = np.array([178//2, 218//2]) displacement_vector = center - eyepad.target_landmarks_[33] # index 33 is the nose-tip nose_centered_landmarks = eyepad.target_landmarks_ + displacement_vector eyepad_cent_nose = EyepadAlign(verbose=1) eyepad_cent_nose.fit_values(target_landmarks=nose_centered_landmarks, target_width=178, target_height=218); ### Evaluate the results img = imageio.imread('test-face.png') img_tr = eyepad.transform(img) img_nose_centered = eyepad_cent_nose.transform(img) fig = plt.figure(figsize=(10, 4)) ax = fig.add_subplot(1, 3, 1) ax.imshow(img) ax.set_title('Original', size=20) ax = fig.add_subplot(1, 3, 2) ax.imshow(img_tr) ax.set_title('Aligned (Average eye \\n position in dataset)', size=15) ax = fig.add_subplot(1, 3, 3) ax.imshow(img_nose_centered) ax.set_title('Aligned (Nose in image center)', size=15) plt.tight_layout() plt.show() API EyepadAlign(verbose=0) Class to align/transform face images to facial landmarks, based on eye alignment. A scaling factor is computed based on distance between the left and right eye, such that the transformed face image will have the same eye distance as a reference face image. A transformation is performed based on the eyes' center point. to align the face based on the reference eye location. Finally, the transformed image is padded with zeros to match the desired final image size. Parameters verbose : int (default=0) Verbose level to display the progress bar and log messages. Setting verbose=1 will print a progress bar upon calling fit_directory . Attributes target_landmarks_ : target landmarks to transform new face images to. Depending on the chosen fit parameters, it can be either (1) assigned to pre-fit shapes, (2) computed from a single face image (3) computed as the mean of face landmarks from all face images in a file directory of face images. eye_distance_ : the distance between left and right eyes in the target landmarks. target_height_ : the height of the transformed output image. target_width_ : the width of the transformed output image. Returns self : object Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/image/eyepad_align/ Methods fit_directory(target_img_dir, target_height, target_width, file_extension='.jpg', pre_check=True) Calculates the average landmarks for all face images in a directory which will then be set as the target landmark set. Arguments target_img_dir : str Directory containing the images target_height : int Expected image height of the images in the directory target_width : int Expected image width of the images in the directory file_extension str (default='.jpg'): File extension of the image files. pre_check Bool (default=True): Checks that each image has the dimensions specificed via target_height and target_width on the whole directory first to identify potential issues that are recommended to be fixed before proceeding. Raises a warning for each image if dimensions differ from the ones specified and expected. Returns self : object fit_image(target_image) Derives facial landmarks from a target image. Arguments target_image : uint8 numpy.array, shape=[width, height, channels] NumPy array representation of the image data. Returns self : object fit_values(target_landmarks, target_width, target_height) Used for determining the eye location from pre-defined landmark arrays, eliminating the need for re-computing the average landmarks on a target image or image directory. Arguments target_landmarks : np.array, shape=(height, width) NumPy array containing the locations of the facial landmarks as determined by mlxtend.image.extract_face_landmarks target_height : int image height target_width : int image width Returns self : object transform(img) transforms a single face image (img) to the target landmarks based on the location of the eyes by scaling, translation and cropping (if needed): (1) Scaling the image so that the distance of the two eyes in the given image (img) matches the distance of the two eyes in the target landmarks. (2) Translation is performed based on the middle point between the two eyes. Arguments img : np.array, shape=(height, width, channels) Input image to be transformed. Returns self : object","title":"EyepadAlign:  align face images based on eye location"},{"location":"user_guide/image/eyepad_align/#eyepadalign-align-face-images-based-on-eye-location","text":"A class to align face images based on eye location. from mlxtend.image import EyepadAlign","title":"EyepadAlign:  align face images based on eye location"},{"location":"user_guide/image/eyepad_align/#overview","text":"The EyepadAlign class align face images to target face landmarks based on the location of the eyes. The facial landmarks are detected using mlxted.image.extract_face_landmarks , which is based on dlib's face landmark detection code [1]. EyepadAlign implements 3 fit methods, fit_image , fit_values , and fit_directory ; once the EyepadAlign object has been fitted via one of these methods (as illustrated in the examples), it can be used to transform a target image.","title":"Overview"},{"location":"user_guide/image/eyepad_align/#references","text":"[1] Kazemi, Vahid, and Josephine Sullivan. \"One millisecond face alignment with an ensemble of regression trees.\" Proceedings of the IEEE Conference on Computer Vision and Pattern Recognition. 2014.","title":"References"},{"location":"user_guide/image/eyepad_align/#example-1-fit_image","text":"This example illustrates the use of EyepadAlign fit to a single target input image (first image in the CelebA dataset ) to another portrait such that the eye position is in the same scale and location as the target image. import imageio from mlxtend.image import EyepadAlign ### Fit EyepadAlign on a single target image eyepad = EyepadAlign() target_image = imageio.imread('celeba-subset/000001.jpg') print('Target image shape: ', target_image.shape) eyepad.fit_image(target_image); Target image shape: (218, 178, 3) ### Transform a test image img = imageio.imread('test-face.png') print('Original Image shape: ', img.shape) img_tr = eyepad.transform(img) print('Transformed Image shape: ', img_tr.shape) Original Image shape: (350, 350, 3) Transformed Image shape: (218, 178, 3) import matplotlib.pyplot as plt fig = plt.figure(figsize=(12, 5)) ax = fig.add_subplot(1, 3, 1) ax.imshow(target_image) ax.set_title('Target', size=20) ax = fig.add_subplot(1, 3, 2) ax.imshow(img) ax.set_title('Original', size=20) ax = fig.add_subplot(1, 3, 3) ax.imshow(img_tr) ax.set_title('Aligned to Target', size=20) plt.show()","title":"Example 1 (fit_image)"},{"location":"user_guide/image/eyepad_align/#example-2-fit_directory","text":"In typical computer vision and deep learning tasks, we are interested in aligning all images in a dataset and across dataset to allow cross-dataset comparisons of face image classifiers. Using the fit_directory method, we can fit EyepadAlign such that the average landmarks are computed from all images in a directory. Then, we can use these average landmarks to transform new images based on the average eye location. import imageio from mlxtend.image import EyepadAlign ### Setting verbosity to 1 will print a progress bar eyepad = EyepadAlign(verbose=1) eyepad.fit_directory(target_img_dir='celeba-subset/', target_width=178, target_height=218, file_extension='.jpg') img = imageio.imread('test-face.png') img_tr = eyepad.transform(img) print('Transformed Image shape: ', img_tr.shape) Pre-Checking directory for consistent image dimensions... 0% [#########] 100% | ETA: 00:00:00 Fitting the average facial landmarks for 9 face images Total time elapsed: 00:00:00 0% [##### ] 100% | ETA: 00:00:00/Users/sebastian/code/mlxtend/mlxtend/image/extract_face_landmarks.py:66: UserWarning: No face detected. warnings.warn('No face detected.') /Users/sebastian/code/mlxtend/mlxtend/image/eyepad_align.py:188: UserWarning: No face detected in image 000004.jpg. Image ignored. % f) 0% [#########] 100% | ETA: 00:00:00 Total time elapsed: 00:00:00 /Users/sebastian/code/mlxtend/mlxtend/image/eyepad_align.py:188: UserWarning: No face detected in image 000003.jpg. Image ignored. % f) Transformed Image shape: (218, 178, 3) Note that a warning is displayed if no face landmarks could be detected. In these cases, the image is skipped and not included in the average landmark computation. E.g., 000003.jpg would be such an example: import matplotlib.pyplot as plt img_no_landmarks = imageio.imread('celeba-subset/000003.jpg') plt.imshow(img_no_landmarks) plt.show() Finally, the an example of aligning the original face image to the average landmarks in the dataset is shown below: fig = plt.figure(figsize=(10, 4)) ax = fig.add_subplot(1, 3, 1) ax.scatter(eyepad.target_landmarks_[:, 0], eyepad.target_landmarks_[:, 1], alpha=0.8) ax.set_title('Target Landmarks', size=20) ax.set_xlim(0, 178) ax.set_ylim(218, 0) ax = fig.add_subplot(1, 3, 2) ax.imshow(img) ax.set_title('Original', size=20) ax = fig.add_subplot(1, 3, 3) ax.imshow(img_tr) ax.set_title('Aligned to Target', size=20) plt.tight_layout() plt.show()","title":"Example 2 (fit_directory)"},{"location":"user_guide/image/eyepad_align/#example-3-fit_values","text":"For more customization, landmarks can also be passed manually. import imageio from mlxtend.image import EyepadAlign import numpy as np celeba_landmarks = np.array([[45, 113], [45, 124], [46, 136], [49, 147], [53, 158], [60, 167], [68, 175], [77, 182], [88, 184], [99, 182], [108, 175], [117, 167], [124, 158], [128, 147], [131, 136], [132, 124], [132, 113], [53, 104], [58, 99], [66, 98], [73, 99], [80, 102], [95, 102], [103, 99], [110, 98], [118, 99], [123, 104], [88, 111], [88, 119], [88, 126], [88, 134], [79, 139], [83, 140], [88, 142], [92, 140], [96, 139], [62, 112], [66, 109], [72, 109], [77, 113], [71, 114], [66, 114], [99, 113], [104, 109], [110, 109], [115, 112], [110, 114], [105, 114], [71, 152], [77, 150], [83, 149], [88, 150], [92, 149], [99, 150], [105, 152], [99, 159], [93, 162], [88, 163], [83, 162], [77, 159], [73, 153], [83, 152], [88, 153], [93, 152], [103, 153], [93, 156], [88, 157], [83, 156]]) eyepad = EyepadAlign(verbose=1) eyepad.fit_values(target_landmarks=celeba_landmarks, target_width=178, target_height=218); import matplotlib.pyplot as plt img = imageio.imread('test-face.png') img_tr = eyepad.transform(img) print('Transformed Image shape: ', img_tr.shape) fig = plt.figure(figsize=(10, 4)) ax = fig.add_subplot(1, 3, 1) ax.scatter(eyepad.target_landmarks_[:, 0], eyepad.target_landmarks_[:, 1], alpha=0.8) ax.set_title('Target Landmarks', size=20) ax.set_xlim(0, 178) ax.set_ylim(218, 0) ax = fig.add_subplot(1, 3, 2) ax.imshow(img) ax.set_title('Original', size=20) ax = fig.add_subplot(1, 3, 3) ax.imshow(img_tr) ax.set_title('Aligned to Target', size=20) plt.tight_layout() plt.show() Transformed Image shape: (218, 178, 3)","title":"Example 3 (fit_values)"},{"location":"user_guide/image/eyepad_align/#example-4-centering-the-nose","text":"In many applications that involve center-cropping, it is helpful to translate the image such that the tip of the nose is located in the center of the image. ### Fit EyepadAlign to dataset to compute average landmarks import imageio from mlxtend.image import EyepadAlign import numpy as np eyepad = EyepadAlign(verbose=1) eyepad.fit_directory(target_img_dir='celeba-subset/', target_width=178, target_height=218, file_extension='.jpg') Pre-Checking directory for consistent image dimensions... 0% [#########] 100% | ETA: 00:00:00 Fitting the average facial landmarks for 9 face images Total time elapsed: 00:00:00 0% [#########] 100% | ETA: 00:00:00 Total time elapsed: 00:00:00 <mlxtend.image.eyepad_align.EyepadAlign at 0x7fa09865d390> ### Use the calculated landmarks to center the nose: center = np.array([178//2, 218//2]) displacement_vector = center - eyepad.target_landmarks_[33] # index 33 is the nose-tip nose_centered_landmarks = eyepad.target_landmarks_ + displacement_vector eyepad_cent_nose = EyepadAlign(verbose=1) eyepad_cent_nose.fit_values(target_landmarks=nose_centered_landmarks, target_width=178, target_height=218); ### Evaluate the results img = imageio.imread('test-face.png') img_tr = eyepad.transform(img) img_nose_centered = eyepad_cent_nose.transform(img) fig = plt.figure(figsize=(10, 4)) ax = fig.add_subplot(1, 3, 1) ax.imshow(img) ax.set_title('Original', size=20) ax = fig.add_subplot(1, 3, 2) ax.imshow(img_tr) ax.set_title('Aligned (Average eye \\n position in dataset)', size=15) ax = fig.add_subplot(1, 3, 3) ax.imshow(img_nose_centered) ax.set_title('Aligned (Nose in image center)', size=15) plt.tight_layout() plt.show()","title":"Example 4 (centering the nose)"},{"location":"user_guide/image/eyepad_align/#api","text":"EyepadAlign(verbose=0) Class to align/transform face images to facial landmarks, based on eye alignment. A scaling factor is computed based on distance between the left and right eye, such that the transformed face image will have the same eye distance as a reference face image. A transformation is performed based on the eyes' center point. to align the face based on the reference eye location. Finally, the transformed image is padded with zeros to match the desired final image size. Parameters verbose : int (default=0) Verbose level to display the progress bar and log messages. Setting verbose=1 will print a progress bar upon calling fit_directory . Attributes target_landmarks_ : target landmarks to transform new face images to. Depending on the chosen fit parameters, it can be either (1) assigned to pre-fit shapes, (2) computed from a single face image (3) computed as the mean of face landmarks from all face images in a file directory of face images. eye_distance_ : the distance between left and right eyes in the target landmarks. target_height_ : the height of the transformed output image. target_width_ : the width of the transformed output image. Returns self : object Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/image/eyepad_align/","title":"API"},{"location":"user_guide/image/eyepad_align/#methods","text":"fit_directory(target_img_dir, target_height, target_width, file_extension='.jpg', pre_check=True) Calculates the average landmarks for all face images in a directory which will then be set as the target landmark set. Arguments target_img_dir : str Directory containing the images target_height : int Expected image height of the images in the directory target_width : int Expected image width of the images in the directory file_extension str (default='.jpg'): File extension of the image files. pre_check Bool (default=True): Checks that each image has the dimensions specificed via target_height and target_width on the whole directory first to identify potential issues that are recommended to be fixed before proceeding. Raises a warning for each image if dimensions differ from the ones specified and expected. Returns self : object fit_image(target_image) Derives facial landmarks from a target image. Arguments target_image : uint8 numpy.array, shape=[width, height, channels] NumPy array representation of the image data. Returns self : object fit_values(target_landmarks, target_width, target_height) Used for determining the eye location from pre-defined landmark arrays, eliminating the need for re-computing the average landmarks on a target image or image directory. Arguments target_landmarks : np.array, shape=(height, width) NumPy array containing the locations of the facial landmarks as determined by mlxtend.image.extract_face_landmarks target_height : int image height target_width : int image width Returns self : object transform(img) transforms a single face image (img) to the target landmarks based on the location of the eyes by scaling, translation and cropping (if needed): (1) Scaling the image so that the distance of the two eyes in the given image (img) matches the distance of the two eyes in the target landmarks. (2) Translation is performed based on the middle point between the two eyes. Arguments img : np.array, shape=(height, width, channels) Input image to be transformed. Returns self : object","title":"Methods"},{"location":"user_guide/math/num_combinations/","text":"num_combinations: combinations for creating subsequences of k elements A function to calculate the number of combinations for creating subsequences of k elements out of a sequence with n elements. from mlxtend.math import num_combinations Overview Combinations are selections of items from a collection regardless of the order in which they appear (in contrast to permutations). For example, let's consider a combination of 3 elements (k=3) from a collection of 5 elements (n=5): collection: {1, 2, 3, 4, 5} combination 1a: {1, 3, 5} combination 1b: {1, 5, 3} combination 1c: {3, 5, 1} ... combination 2: {1, 3, 4} In the example above the combinations 1a, 1b, and 1c, are the \"same combination\" and counted as \"1 possible way to combine items 1, 3, and 5\" -- in combinations, the order does not matter. The number of ways to combine elements ( without replacement ) from a collection with size n into subsets of size k is computed via the binomial coefficient (\" n choose k \"): \\begin{pmatrix} n \\\\ k \\end{pmatrix} = \\frac{n(n-1)\\ldots(n-k+1)}{k(k-1)\\dots1} = \\frac{n!}{k!(n-k)!} To compute the number of combinations with replacement , the following, alternative equation is used (\" n multichoose k \"): \\begin{pmatrix} n \\\\ k \\end{pmatrix} = \\begin{pmatrix} n + k -1 \\\\ k \\end{pmatrix} References https://en.wikipedia.org/wiki/Combination Example 1 - Compute the number of combinations from mlxtend.math import num_combinations c = num_combinations(n=20, k=8, with_replacement=False) print('Number of ways to combine 20 elements' ' into 8 subelements: %d' % c) Number of ways to combine 20 elements into 8 subelements: 125970 from mlxtend.math import num_combinations c = num_combinations(n=20, k=8, with_replacement=True) print('Number of ways to combine 20 elements' ' into 8 subelements (with replacement): %d' % c) Number of ways to combine 20 elements into 8 subelements (with replacement): 2220075 Example 2 - A progress tracking use-case It is often quite useful to track the progress of a computational expensive tasks to estimate its runtime. Here, the num_combination function can be used to compute the maximum number of loops of a combinations iterable from itertools: import itertools import sys import time from mlxtend.math import num_combinations items = {1, 2, 3, 4, 5, 6, 7, 8} max_iter = num_combinations(n=len(items), k=3, with_replacement=False) for idx, i in enumerate(itertools.combinations(items, r=3)): # do some computation with itemset i time.sleep(0.1) sys.stdout.write('\\rProgress: %d/%d' % (idx + 1, max_iter)) sys.stdout.flush() Progress: 56/56 API num_combinations(n, k, with_replacement=False) Function to calculate the number of possible combinations. Parameters n : int Total number of items. k : int Number of elements of the target itemset. with_replacement : bool (default: False) Allows repeated elements if True. Returns comb : int Number of possible combinations. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/math/num_combinations/","title":"num_combinations: combinations for creating subsequences of *k* elements"},{"location":"user_guide/math/num_combinations/#num_combinations-combinations-for-creating-subsequences-of-k-elements","text":"A function to calculate the number of combinations for creating subsequences of k elements out of a sequence with n elements. from mlxtend.math import num_combinations","title":"num_combinations: combinations for creating subsequences of k elements"},{"location":"user_guide/math/num_combinations/#overview","text":"Combinations are selections of items from a collection regardless of the order in which they appear (in contrast to permutations). For example, let's consider a combination of 3 elements (k=3) from a collection of 5 elements (n=5): collection: {1, 2, 3, 4, 5} combination 1a: {1, 3, 5} combination 1b: {1, 5, 3} combination 1c: {3, 5, 1} ... combination 2: {1, 3, 4} In the example above the combinations 1a, 1b, and 1c, are the \"same combination\" and counted as \"1 possible way to combine items 1, 3, and 5\" -- in combinations, the order does not matter. The number of ways to combine elements ( without replacement ) from a collection with size n into subsets of size k is computed via the binomial coefficient (\" n choose k \"): \\begin{pmatrix} n \\\\ k \\end{pmatrix} = \\frac{n(n-1)\\ldots(n-k+1)}{k(k-1)\\dots1} = \\frac{n!}{k!(n-k)!} To compute the number of combinations with replacement , the following, alternative equation is used (\" n multichoose k \"): \\begin{pmatrix} n \\\\ k \\end{pmatrix} = \\begin{pmatrix} n + k -1 \\\\ k \\end{pmatrix}","title":"Overview"},{"location":"user_guide/math/num_combinations/#references","text":"https://en.wikipedia.org/wiki/Combination","title":"References"},{"location":"user_guide/math/num_combinations/#example-1-compute-the-number-of-combinations","text":"from mlxtend.math import num_combinations c = num_combinations(n=20, k=8, with_replacement=False) print('Number of ways to combine 20 elements' ' into 8 subelements: %d' % c) Number of ways to combine 20 elements into 8 subelements: 125970 from mlxtend.math import num_combinations c = num_combinations(n=20, k=8, with_replacement=True) print('Number of ways to combine 20 elements' ' into 8 subelements (with replacement): %d' % c) Number of ways to combine 20 elements into 8 subelements (with replacement): 2220075","title":"Example 1 - Compute the number of combinations"},{"location":"user_guide/math/num_combinations/#example-2-a-progress-tracking-use-case","text":"It is often quite useful to track the progress of a computational expensive tasks to estimate its runtime. Here, the num_combination function can be used to compute the maximum number of loops of a combinations iterable from itertools: import itertools import sys import time from mlxtend.math import num_combinations items = {1, 2, 3, 4, 5, 6, 7, 8} max_iter = num_combinations(n=len(items), k=3, with_replacement=False) for idx, i in enumerate(itertools.combinations(items, r=3)): # do some computation with itemset i time.sleep(0.1) sys.stdout.write('\\rProgress: %d/%d' % (idx + 1, max_iter)) sys.stdout.flush() Progress: 56/56","title":"Example 2 - A progress tracking use-case"},{"location":"user_guide/math/num_combinations/#api","text":"num_combinations(n, k, with_replacement=False) Function to calculate the number of possible combinations. Parameters n : int Total number of items. k : int Number of elements of the target itemset. with_replacement : bool (default: False) Allows repeated elements if True. Returns comb : int Number of possible combinations. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/math/num_combinations/","title":"API"},{"location":"user_guide/math/num_permutations/","text":"num_permutations: number of permutations for creating subsequences of k elements A function to calculate the number of permutations for creating subsequences of k elements out of a sequence with n elements. from mlxtend.math import num_permutations Overview Permutations are selections of items from a collection with regard to the order in which they appear (in contrast to combinations). For example, let's consider a permutation of 3 elements (k=3) from a collection of 5 elements (n=5): collection: {1, 2, 3, 4, 5} combination 1a: {1, 3, 5} combination 1b: {1, 5, 3} combination 1c: {3, 5, 1} ... combination 2: {1, 3, 4} In the example above the permutations 1a, 1b, and 1c, are the \"same combination\" but distinct permutations -- in combinations, the order does not matter, but in permutation it does matter. The number of ways to combine elements ( without replacement ) from a collection with size n into subsets of size k is computed via the binomial coefficient (\" n choose k \"): k!\\begin{pmatrix} n \\\\ k \\end{pmatrix} = k! \\cdot \\frac{n!}{k!(n-k)!} = \\frac{n!}{(n-k)!} To compute the number of permutations with replacement , we simply need to compute n^k . References https://en.wikipedia.org/wiki/Permutation Example 1 - Compute the number of permutations from mlxtend.math import num_permutations c = num_permutations(n=20, k=8, with_replacement=False) print('Number of ways to permute 20 elements' ' into 8 subelements: %d' % c) Number of ways to permute 20 elements into 8 subelements: 5079110400 from mlxtend.math import num_permutations c = num_permutations(n=20, k=8, with_replacement=True) print('Number of ways to combine 20 elements' ' into 8 subelements (with replacement): %d' % c) Number of ways to combine 20 elements into 8 subelements (with replacement): 25600000000 Example 2 - A progress tracking use-case It is often quite useful to track the progress of a computational expensive tasks to estimate its runtime. Here, the num_combination function can be used to compute the maximum number of loops of a permutations iterable from itertools: import itertools import sys import time from mlxtend.math import num_permutations items = {1, 2, 3, 4, 5, 6, 7, 8} max_iter = num_permutations(n=len(items), k=3, with_replacement=False) for idx, i in enumerate(itertools.permutations(items, r=3)): # do some computation with itemset i time.sleep(0.01) sys.stdout.write('\\rProgress: %d/%d' % (idx + 1, max_iter)) sys.stdout.flush() Progress: 336/336 API num_permutations(n, k, with_replacement=False) Function to calculate the number of possible permutations. Parameters n : int Total number of items. k : int Number of elements of the target itemset. with_replacement : bool Allows repeated elements if True. Returns permut : int Number of possible permutations. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/math/num_permutations/","title":"num_permutations: number of permutations for creating subsequences of *k* elements"},{"location":"user_guide/math/num_permutations/#num_permutations-number-of-permutations-for-creating-subsequences-of-k-elements","text":"A function to calculate the number of permutations for creating subsequences of k elements out of a sequence with n elements. from mlxtend.math import num_permutations","title":"num_permutations: number of permutations for creating subsequences of k elements"},{"location":"user_guide/math/num_permutations/#overview","text":"Permutations are selections of items from a collection with regard to the order in which they appear (in contrast to combinations). For example, let's consider a permutation of 3 elements (k=3) from a collection of 5 elements (n=5): collection: {1, 2, 3, 4, 5} combination 1a: {1, 3, 5} combination 1b: {1, 5, 3} combination 1c: {3, 5, 1} ... combination 2: {1, 3, 4} In the example above the permutations 1a, 1b, and 1c, are the \"same combination\" but distinct permutations -- in combinations, the order does not matter, but in permutation it does matter. The number of ways to combine elements ( without replacement ) from a collection with size n into subsets of size k is computed via the binomial coefficient (\" n choose k \"): k!\\begin{pmatrix} n \\\\ k \\end{pmatrix} = k! \\cdot \\frac{n!}{k!(n-k)!} = \\frac{n!}{(n-k)!} To compute the number of permutations with replacement , we simply need to compute n^k .","title":"Overview"},{"location":"user_guide/math/num_permutations/#references","text":"https://en.wikipedia.org/wiki/Permutation","title":"References"},{"location":"user_guide/math/num_permutations/#example-1-compute-the-number-of-permutations","text":"from mlxtend.math import num_permutations c = num_permutations(n=20, k=8, with_replacement=False) print('Number of ways to permute 20 elements' ' into 8 subelements: %d' % c) Number of ways to permute 20 elements into 8 subelements: 5079110400 from mlxtend.math import num_permutations c = num_permutations(n=20, k=8, with_replacement=True) print('Number of ways to combine 20 elements' ' into 8 subelements (with replacement): %d' % c) Number of ways to combine 20 elements into 8 subelements (with replacement): 25600000000","title":"Example 1 - Compute the number of permutations"},{"location":"user_guide/math/num_permutations/#example-2-a-progress-tracking-use-case","text":"It is often quite useful to track the progress of a computational expensive tasks to estimate its runtime. Here, the num_combination function can be used to compute the maximum number of loops of a permutations iterable from itertools: import itertools import sys import time from mlxtend.math import num_permutations items = {1, 2, 3, 4, 5, 6, 7, 8} max_iter = num_permutations(n=len(items), k=3, with_replacement=False) for idx, i in enumerate(itertools.permutations(items, r=3)): # do some computation with itemset i time.sleep(0.01) sys.stdout.write('\\rProgress: %d/%d' % (idx + 1, max_iter)) sys.stdout.flush() Progress: 336/336","title":"Example 2 - A progress tracking use-case"},{"location":"user_guide/math/num_permutations/#api","text":"num_permutations(n, k, with_replacement=False) Function to calculate the number of possible permutations. Parameters n : int Total number of items. k : int Number of elements of the target itemset. with_replacement : bool Allows repeated elements if True. Returns permut : int Number of possible permutations. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/math/num_permutations/","title":"API"},{"location":"user_guide/math/vectorspace_dimensionality/","text":"vectorspace_dimensionality: compute the number of dimensions that a set of vectors spans A function to compute the number of dimensions a set of vectors (arranged as columns in a matrix) spans. from mlxtend.math import vectorspace_dimensionality Overview Given a set of vectors, arranged as columns in a matrix, the vectorspace_dimensionality computes the number of dimensions (i.e., hyper-volume) that the vectorspace spans using the Gram-Schmidt process [1]. In particular, since the Gram-Schmidt process yields vectors that are zero or normalized to 1 (i.e., an orthonormal vectorset if the input was a set of linearly independent vectors), the sum of the vector norms corresponds to the number of dimensions of a vectorset. References [1] https://en.wikipedia.org/wiki/Gram\u2013Schmidt_process Example 1 - Compute the dimensions of a vectorspace Let's assume we have the two basis vectors x=[1 \\;\\;\\; 0]^T and y=[0\\;\\;\\; 1]^T as columns in a matrix. Due to the linear independence of the two vectors, the space that they span is naturally a plane (2D space): import numpy as np from mlxtend.math import vectorspace_dimensionality a = np.array([[1, 0], [0, 1]]) vectorspace_dimensionality(a) 2 However, if one vector is a linear combination of the other, it's intuitive to see that the space the vectorset describes is merely a line, aka a 1D space: b = np.array([[1, 2], [0, 0]]) vectorspace_dimensionality(a) 2 If 3 vectors are all linearly independent of each other, the dimensionality of the vector space is a volume (i.e., a 3D space): d = np.array([[1, 9, 1], [3, 2, 2], [5, 4, 3]]) vectorspace_dimensionality(d) 3 Again, if a pair of vectors is linearly dependent (here: the 1st and the 2nd row), this reduces the dimensionality by 1: c = np.array([[1, 2, 1], [3, 6, 2], [5, 10, 3]]) vectorspace_dimensionality(c) 2 API vectorspace_dimensionality(ary) Computes the hyper-volume spanned by a vector set Parameters ary : array-like, shape=[num_vectors, num_vectors] An orthogonal set of vectors (arranged as columns in a matrix) Returns dimensions : int An integer indicating the \"dimensionality\" hyper-volume spanned by the vector set Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/math/vectorspace_dimensionality/","title":"vectorspace_dimensionality: compute the number of dimensions that a set of vectors spans"},{"location":"user_guide/math/vectorspace_dimensionality/#vectorspace_dimensionality-compute-the-number-of-dimensions-that-a-set-of-vectors-spans","text":"A function to compute the number of dimensions a set of vectors (arranged as columns in a matrix) spans. from mlxtend.math import vectorspace_dimensionality","title":"vectorspace_dimensionality: compute the number of dimensions that a set of vectors spans"},{"location":"user_guide/math/vectorspace_dimensionality/#overview","text":"Given a set of vectors, arranged as columns in a matrix, the vectorspace_dimensionality computes the number of dimensions (i.e., hyper-volume) that the vectorspace spans using the Gram-Schmidt process [1]. In particular, since the Gram-Schmidt process yields vectors that are zero or normalized to 1 (i.e., an orthonormal vectorset if the input was a set of linearly independent vectors), the sum of the vector norms corresponds to the number of dimensions of a vectorset.","title":"Overview"},{"location":"user_guide/math/vectorspace_dimensionality/#references","text":"[1] https://en.wikipedia.org/wiki/Gram\u2013Schmidt_process","title":"References"},{"location":"user_guide/math/vectorspace_dimensionality/#example-1-compute-the-dimensions-of-a-vectorspace","text":"Let's assume we have the two basis vectors x=[1 \\;\\;\\; 0]^T and y=[0\\;\\;\\; 1]^T as columns in a matrix. Due to the linear independence of the two vectors, the space that they span is naturally a plane (2D space): import numpy as np from mlxtend.math import vectorspace_dimensionality a = np.array([[1, 0], [0, 1]]) vectorspace_dimensionality(a) 2 However, if one vector is a linear combination of the other, it's intuitive to see that the space the vectorset describes is merely a line, aka a 1D space: b = np.array([[1, 2], [0, 0]]) vectorspace_dimensionality(a) 2 If 3 vectors are all linearly independent of each other, the dimensionality of the vector space is a volume (i.e., a 3D space): d = np.array([[1, 9, 1], [3, 2, 2], [5, 4, 3]]) vectorspace_dimensionality(d) 3 Again, if a pair of vectors is linearly dependent (here: the 1st and the 2nd row), this reduces the dimensionality by 1: c = np.array([[1, 2, 1], [3, 6, 2], [5, 10, 3]]) vectorspace_dimensionality(c) 2","title":"Example 1 - Compute the dimensions of a vectorspace"},{"location":"user_guide/math/vectorspace_dimensionality/#api","text":"vectorspace_dimensionality(ary) Computes the hyper-volume spanned by a vector set Parameters ary : array-like, shape=[num_vectors, num_vectors] An orthogonal set of vectors (arranged as columns in a matrix) Returns dimensions : int An integer indicating the \"dimensionality\" hyper-volume spanned by the vector set Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/math/vectorspace_dimensionality/","title":"API"},{"location":"user_guide/math/vectorspace_orthonormalization/","text":"vectorspace_orthonormalization: Converts a set of linearly independent vectors to a set of orthonormal basis vectors A function that converts a set of linearly independent vectors to a set of orthonormal basis vectors. from mlxtend.math import vectorspace_orthonormalization Overview The vectorspace_orthonormalization converts a set linearly independent vectors to a set of orthonormal basis vectors using the Gram-Schmidt process [1]. References [1] https://en.wikipedia.org/wiki/Gram\u2013Schmidt_process Example 1 - Convert a set of vector to an orthonormal basis Note that to convert a set of linearly independent vectors into a set of orthonormal basis vectors, the vectorspace_orthonormalization function expects the vectors to be arranged as columns of a matrix (here: NumPy array). Please keep in mind that the vectorspace_orthonormalization function also works for non-linearly independent vector sets; however, the resulting vectorset won't be orthonormal as a result. An easy way to check whether all vectors in the input set are linearly independent is to use the numpy.linalg.det (determinant) function. import numpy as np from mlxtend.math import vectorspace_orthonormalization a = np.array([[2, 0, 4, 12], [0, 2, 16, 4], [4, 16, 6, 2], [2, -12, 4, 6]]) s = '' if np.linalg.det(a) == 0.0: s = ' not' print('Input vectors are%s linearly independent' % s) vectorspace_orthonormalization(a) Input vectors are linearly independent array([[ 0.40824829, -0.1814885 , 0.04982278, 0.89325973], [ 0. , 0.1088931 , 0.99349591, -0.03328918], [ 0.81649658, 0.50816781, -0.06462163, -0.26631346], [ 0.40824829, -0.83484711, 0.07942048, -0.36063281]]) Note that scaling the inputs equally by a factor should leave the results unchanged: vectorspace_orthonormalization(a/2) array([[ 0.40824829, -0.1814885 , 0.04982278, 0.89325973], [ 0. , 0.1088931 , 0.99349591, -0.03328918], [ 0.81649658, 0.50816781, -0.06462163, -0.26631346], [ 0.40824829, -0.83484711, 0.07942048, -0.36063281]]) However, in case of linear dependence (the second column is a linear combination of the first column in the example below), the vector elements of one of the dependent vectors will become zero. (For a pair of linear dependent vectors, the one with the larger column index will be the one that's zero-ed.) a[:, 1] = a[:, 0] * 2 vectorspace_orthonormalization(a) array([[ 0.40824829, 0. , 0.04155858, 0.82364839], [ 0. , 0. , 0.99740596, -0.06501108], [ 0.81649658, 0. , -0.04155858, -0.52008861], [ 0.40824829, 0. , 0.04155858, 0.21652883]]) API vectorspace_orthonormalization(ary, eps=1e-13) Transforms a set of column vectors to a orthonormal basis. Given a set of orthogonal vectors, this functions converts such column vectors, arranged in a matrix, into orthonormal basis vectors. Parameters ary : array-like, shape=[num_vectors, num_vectors] An orthogonal set of vectors (arranged as columns in a matrix) eps : float (default: 1e-13) A small tolerance value to determine whether the vector norm is zero or not. Returns arr : array-like, shape=[num_vectors, num_vectors] An orthonormal set of vectors (arranged as columns) Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/math/vectorspace_orthonormalization/","title":"vectorspace_orthonormalization: Converts a set of linearly independent vectors to a set of orthonormal basis vectors"},{"location":"user_guide/math/vectorspace_orthonormalization/#vectorspace_orthonormalization-converts-a-set-of-linearly-independent-vectors-to-a-set-of-orthonormal-basis-vectors","text":"A function that converts a set of linearly independent vectors to a set of orthonormal basis vectors. from mlxtend.math import vectorspace_orthonormalization","title":"vectorspace_orthonormalization: Converts a set of linearly independent vectors to a set of orthonormal basis vectors"},{"location":"user_guide/math/vectorspace_orthonormalization/#overview","text":"The vectorspace_orthonormalization converts a set linearly independent vectors to a set of orthonormal basis vectors using the Gram-Schmidt process [1].","title":"Overview"},{"location":"user_guide/math/vectorspace_orthonormalization/#references","text":"[1] https://en.wikipedia.org/wiki/Gram\u2013Schmidt_process","title":"References"},{"location":"user_guide/math/vectorspace_orthonormalization/#example-1-convert-a-set-of-vector-to-an-orthonormal-basis","text":"Note that to convert a set of linearly independent vectors into a set of orthonormal basis vectors, the vectorspace_orthonormalization function expects the vectors to be arranged as columns of a matrix (here: NumPy array). Please keep in mind that the vectorspace_orthonormalization function also works for non-linearly independent vector sets; however, the resulting vectorset won't be orthonormal as a result. An easy way to check whether all vectors in the input set are linearly independent is to use the numpy.linalg.det (determinant) function. import numpy as np from mlxtend.math import vectorspace_orthonormalization a = np.array([[2, 0, 4, 12], [0, 2, 16, 4], [4, 16, 6, 2], [2, -12, 4, 6]]) s = '' if np.linalg.det(a) == 0.0: s = ' not' print('Input vectors are%s linearly independent' % s) vectorspace_orthonormalization(a) Input vectors are linearly independent array([[ 0.40824829, -0.1814885 , 0.04982278, 0.89325973], [ 0. , 0.1088931 , 0.99349591, -0.03328918], [ 0.81649658, 0.50816781, -0.06462163, -0.26631346], [ 0.40824829, -0.83484711, 0.07942048, -0.36063281]]) Note that scaling the inputs equally by a factor should leave the results unchanged: vectorspace_orthonormalization(a/2) array([[ 0.40824829, -0.1814885 , 0.04982278, 0.89325973], [ 0. , 0.1088931 , 0.99349591, -0.03328918], [ 0.81649658, 0.50816781, -0.06462163, -0.26631346], [ 0.40824829, -0.83484711, 0.07942048, -0.36063281]]) However, in case of linear dependence (the second column is a linear combination of the first column in the example below), the vector elements of one of the dependent vectors will become zero. (For a pair of linear dependent vectors, the one with the larger column index will be the one that's zero-ed.) a[:, 1] = a[:, 0] * 2 vectorspace_orthonormalization(a) array([[ 0.40824829, 0. , 0.04155858, 0.82364839], [ 0. , 0. , 0.99740596, -0.06501108], [ 0.81649658, 0. , -0.04155858, -0.52008861], [ 0.40824829, 0. , 0.04155858, 0.21652883]])","title":"Example 1 - Convert a set of vector to an orthonormal basis"},{"location":"user_guide/math/vectorspace_orthonormalization/#api","text":"vectorspace_orthonormalization(ary, eps=1e-13) Transforms a set of column vectors to a orthonormal basis. Given a set of orthogonal vectors, this functions converts such column vectors, arranged in a matrix, into orthonormal basis vectors. Parameters ary : array-like, shape=[num_vectors, num_vectors] An orthogonal set of vectors (arranged as columns in a matrix) eps : float (default: 1e-13) A small tolerance value to determine whether the vector norm is zero or not. Returns arr : array-like, shape=[num_vectors, num_vectors] An orthonormal set of vectors (arranged as columns) Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/math/vectorspace_orthonormalization/","title":"API"},{"location":"user_guide/plotting/category_scatter/","text":"Scategory_scatter: Create a scatterplot with categories in different colors A function to quickly produce a scatter plot colored by categories from a pandas DataFrame or NumPy ndarray object. from mlxtend.general import category_scatter Overview References - Example 1 - Category Scatter from Pandas DataFrames import pandas as pd from io import StringIO csvfile = \"\"\"label,x,y class1,10.0,8.04 class1,10.5,7.30 class2,8.3,5.5 class2,8.1,5.9 class3,3.5,3.5 class3,3.8,5.1\"\"\" df = pd.read_csv(StringIO(csvfile)) df label x y 0 class1 10.0 8.04 1 class1 10.5 7.30 2 class2 8.3 5.50 3 class2 8.1 5.90 4 class3 3.5 3.50 5 class3 3.8 5.10 Plotting the data where the categories are determined by the unique values in the label column label_col . The x and y values are simply the column names of the DataFrame that we want to plot. import matplotlib.pyplot as plt from mlxtend.plotting import category_scatter fig = category_scatter(x='x', y='y', label_col='label', data=df, legend_loc='upper left') Example 2 - Category Scatter from NumPy Arrays import numpy as np from io import BytesIO csvfile = \"\"\"1,10.0,8.04 1,10.5,7.30 2,8.3,5.5 2,8.1,5.9 3,3.5,3.5 3,3.8,5.1\"\"\" ary = np.genfromtxt(BytesIO(csvfile.encode()), delimiter=',') ary array([[ 1. , 10. , 8.04], [ 1. , 10.5 , 7.3 ], [ 2. , 8.3 , 5.5 ], [ 2. , 8.1 , 5.9 ], [ 3. , 3.5 , 3.5 ], [ 3. , 3.8 , 5.1 ]]) Now, pretending that the first column represents the labels, and the second and third column represent the x and y values, respectively. import matplotlib.pyplot as plt from mlxtend.plotting import category_scatter fix = category_scatter(x=1, y=2, label_col=0, data=ary, legend_loc='upper left') API category_scatter(x, y, label_col, data, markers='sxo^v', colors=('blue', 'green', 'red', 'purple', 'gray', 'cyan'), alpha=0.7, markersize=20.0, legend_loc='best') Scatter plot to plot categories in different colors/markerstyles. Parameters x : str or int DataFrame column name of the x-axis values or integer for the numpy ndarray column index. y : str DataFrame column name of the y-axis values or integer for the numpy ndarray column index data : Pandas DataFrame object or NumPy ndarray. markers : str Markers that are cycled through the label category. colors : tuple Colors that are cycled through the label category. alpha : float (default: 0.7) Parameter to control the transparency. markersize : float (default` : 20.0) Parameter to control the marker size. legend_loc : str (default: 'best') Location of the plot legend {best, upper left, upper right, lower left, lower right} No legend if legend_loc=False Returns fig : matplotlig.pyplot figure object Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/plotting/category_scatter/","title":"Scategory_scatter: Create a scatterplot with categories in different colors"},{"location":"user_guide/plotting/category_scatter/#scategory_scatter-create-a-scatterplot-with-categories-in-different-colors","text":"A function to quickly produce a scatter plot colored by categories from a pandas DataFrame or NumPy ndarray object. from mlxtend.general import category_scatter","title":"Scategory_scatter: Create a scatterplot with categories in different colors"},{"location":"user_guide/plotting/category_scatter/#overview","text":"","title":"Overview"},{"location":"user_guide/plotting/category_scatter/#references","text":"-","title":"References"},{"location":"user_guide/plotting/category_scatter/#example-1-category-scatter-from-pandas-dataframes","text":"import pandas as pd from io import StringIO csvfile = \"\"\"label,x,y class1,10.0,8.04 class1,10.5,7.30 class2,8.3,5.5 class2,8.1,5.9 class3,3.5,3.5 class3,3.8,5.1\"\"\" df = pd.read_csv(StringIO(csvfile)) df label x y 0 class1 10.0 8.04 1 class1 10.5 7.30 2 class2 8.3 5.50 3 class2 8.1 5.90 4 class3 3.5 3.50 5 class3 3.8 5.10 Plotting the data where the categories are determined by the unique values in the label column label_col . The x and y values are simply the column names of the DataFrame that we want to plot. import matplotlib.pyplot as plt from mlxtend.plotting import category_scatter fig = category_scatter(x='x', y='y', label_col='label', data=df, legend_loc='upper left')","title":"Example 1 - Category Scatter from Pandas DataFrames"},{"location":"user_guide/plotting/category_scatter/#example-2-category-scatter-from-numpy-arrays","text":"import numpy as np from io import BytesIO csvfile = \"\"\"1,10.0,8.04 1,10.5,7.30 2,8.3,5.5 2,8.1,5.9 3,3.5,3.5 3,3.8,5.1\"\"\" ary = np.genfromtxt(BytesIO(csvfile.encode()), delimiter=',') ary array([[ 1. , 10. , 8.04], [ 1. , 10.5 , 7.3 ], [ 2. , 8.3 , 5.5 ], [ 2. , 8.1 , 5.9 ], [ 3. , 3.5 , 3.5 ], [ 3. , 3.8 , 5.1 ]]) Now, pretending that the first column represents the labels, and the second and third column represent the x and y values, respectively. import matplotlib.pyplot as plt from mlxtend.plotting import category_scatter fix = category_scatter(x=1, y=2, label_col=0, data=ary, legend_loc='upper left')","title":"Example 2 - Category Scatter from NumPy Arrays"},{"location":"user_guide/plotting/category_scatter/#api","text":"category_scatter(x, y, label_col, data, markers='sxo^v', colors=('blue', 'green', 'red', 'purple', 'gray', 'cyan'), alpha=0.7, markersize=20.0, legend_loc='best') Scatter plot to plot categories in different colors/markerstyles. Parameters x : str or int DataFrame column name of the x-axis values or integer for the numpy ndarray column index. y : str DataFrame column name of the y-axis values or integer for the numpy ndarray column index data : Pandas DataFrame object or NumPy ndarray. markers : str Markers that are cycled through the label category. colors : tuple Colors that are cycled through the label category. alpha : float (default: 0.7) Parameter to control the transparency. markersize : float (default` : 20.0) Parameter to control the marker size. legend_loc : str (default: 'best') Location of the plot legend {best, upper left, upper right, lower left, lower right} No legend if legend_loc=False Returns fig : matplotlig.pyplot figure object Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/plotting/category_scatter/","title":"API"},{"location":"user_guide/plotting/checkerboard_plot/","text":"checkerboard_plot: Create a checkerboard plot in matplotlib Function to plot a checkerboard plot / heat map via matplotlib from mlxtend.plotting import checkerboard_plot Overview Function to plot a checkerboard plot / heat map via matplotlib. References - Example 1 - Default from mlxtend.plotting import checkerboard_plot import matplotlib.pyplot as plt import numpy as np ary = np.random.random((5, 4)) brd = checkerboard_plot(ary) plt.show() Example 2 - Changing colors and labels from mlxtend.plotting import checkerboard_plot import matplotlib.pyplot as plt import numpy as np checkerboard_plot(ary, col_labels=['abc', 'def', 'ghi', 'jkl'], row_labels=['sample %d' % i for i in range(1, 6)], cell_colors=['skyblue', 'whitesmoke'], font_colors=['black', 'black'], figsize=(4.5, 5)) plt.show() API checkerboard_plot(ary, cell_colors=('white', 'black'), font_colors=('black', 'white'), fmt='%.1f', figsize=None, row_labels=None, col_labels=None, fontsize=None) Plot a checkerboard table / heatmap via matplotlib. Parameters ary : array-like, shape = [n, m] A 2D Nnumpy array. cell_colors : tuple or list (default: ('white', 'black')) Tuple or list containing the two colors of the checkerboard pattern. font_colors : tuple or list (default: ('black', 'white')) Font colors corresponding to the cell colors. figsize : tuple (default: (2.5, 2.5)) Height and width of the figure fmt : str (default: '%.1f') Python string formatter for cell values. The default '%.1f' results in floats with 1 digit after the decimal point. Use '%d' to show numbers as integers. row_labels : list (default: None) List of the row labels. Uses the array row indices 0 to n by default. col_labels : list (default: None) List of the column labels. Uses the array column indices 0 to m by default. fontsize : int (default: None) Specifies the font size of the checkerboard table. Uses matplotlib's default if None. Returns fig : matplotlib Figure object. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/plotting/checkerboard_plot/","title":"checkerboard_plot: Create a checkerboard plot in matplotlib"},{"location":"user_guide/plotting/checkerboard_plot/#checkerboard_plot-create-a-checkerboard-plot-in-matplotlib","text":"Function to plot a checkerboard plot / heat map via matplotlib from mlxtend.plotting import checkerboard_plot","title":"checkerboard_plot: Create a checkerboard plot in matplotlib"},{"location":"user_guide/plotting/checkerboard_plot/#overview","text":"Function to plot a checkerboard plot / heat map via matplotlib.","title":"Overview"},{"location":"user_guide/plotting/checkerboard_plot/#references","text":"-","title":"References"},{"location":"user_guide/plotting/checkerboard_plot/#example-1-default","text":"from mlxtend.plotting import checkerboard_plot import matplotlib.pyplot as plt import numpy as np ary = np.random.random((5, 4)) brd = checkerboard_plot(ary) plt.show()","title":"Example 1 - Default"},{"location":"user_guide/plotting/checkerboard_plot/#example-2-changing-colors-and-labels","text":"from mlxtend.plotting import checkerboard_plot import matplotlib.pyplot as plt import numpy as np checkerboard_plot(ary, col_labels=['abc', 'def', 'ghi', 'jkl'], row_labels=['sample %d' % i for i in range(1, 6)], cell_colors=['skyblue', 'whitesmoke'], font_colors=['black', 'black'], figsize=(4.5, 5)) plt.show()","title":"Example 2 - Changing colors and labels"},{"location":"user_guide/plotting/checkerboard_plot/#api","text":"checkerboard_plot(ary, cell_colors=('white', 'black'), font_colors=('black', 'white'), fmt='%.1f', figsize=None, row_labels=None, col_labels=None, fontsize=None) Plot a checkerboard table / heatmap via matplotlib. Parameters ary : array-like, shape = [n, m] A 2D Nnumpy array. cell_colors : tuple or list (default: ('white', 'black')) Tuple or list containing the two colors of the checkerboard pattern. font_colors : tuple or list (default: ('black', 'white')) Font colors corresponding to the cell colors. figsize : tuple (default: (2.5, 2.5)) Height and width of the figure fmt : str (default: '%.1f') Python string formatter for cell values. The default '%.1f' results in floats with 1 digit after the decimal point. Use '%d' to show numbers as integers. row_labels : list (default: None) List of the row labels. Uses the array row indices 0 to n by default. col_labels : list (default: None) List of the column labels. Uses the array column indices 0 to m by default. fontsize : int (default: None) Specifies the font size of the checkerboard table. Uses matplotlib's default if None. Returns fig : matplotlib Figure object. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/plotting/checkerboard_plot/","title":"API"},{"location":"user_guide/plotting/ecdf/","text":"ecdf: Create an empirical cumulative distribution function plot A function to conveniently plot an empirical cumulative distribution function. from mlxtend.ecdf import ecdf Overview A function to conveniently plot an empirical cumulative distribution function (ECDF) and adding percentile thresholds for exploratory data analysis. References - Example 1 - ECDF from mlxtend.data import iris_data from mlxtend.plotting import ecdf import matplotlib.pyplot as plt X, y = iris_data() ax, _, _ = ecdf(x=X[:, 0], x_label='sepal length (cm)') plt.show() Example 2 - Multiple ECDFs from mlxtend.data import iris_data from mlxtend.plotting import ecdf import matplotlib.pyplot as plt X, y = iris_data() # first ecdf x1 = X[:, 0] ax, _, _ = ecdf(x1, x_label='cm') # second ecdf x2 = X[:, 1] ax, _, _ = ecdf(x2, ax=ax) plt.legend(['sepal length', 'sepal width']) plt.show() Example 3 - ECDF with Percentile Thresholds from mlxtend.data import iris_data from mlxtend.plotting import ecdf import matplotlib.pyplot as plt X, y = iris_data() ax, threshold, count = ecdf(x=X[:, 0], x_label='sepal length (cm)', percentile=0.8) plt.show() print('Feature threshold at the 80th percentile:', threshold) print('Number of samples below the threshold:', count) Feature threshold at the 80th percentile: 6.5 Number of samples below the threshold: 120 API ecdf(x, y_label='ECDF', x_label=None, ax=None, percentile=None, ecdf_color=None, ecdf_marker='o', percentile_color='black', percentile_linestyle='--') Plots an Empirical Cumulative Distribution Function Parameters x : array or list, shape=[n_samples,] Array-like object containing the feature values y_label : str (default='ECDF') Text label for the y-axis x_label : str (default=None) Text label for the x-axis ax : matplotlib.axes.Axes (default: None) An existing matplotlib Axes. Creates one if ax=None percentile : float (default=None) Float between 0 and 1 for plotting a percentile threshold line ecdf_color : matplotlib color (default=None) Color for the ECDF plot; uses matplotlib defaults if None ecdf_marker : matplotlib marker (default='o') Marker style for the ECDF plot percentile_color : matplotlib color (default='black') Color for the percentile threshold if percentile is not None percentile_linestyle : matplotlib linestyle (default='--') Line style for the percentile threshold if percentile is not None Returns ax : matplotlib.axes.Axes object percentile_threshold : float Feature threshold at the percentile or None if percentile=None percentile_count : Number of if percentile is not None Number of samples that have a feature less or equal than the feature threshold at a percentile threshold or None if percentile=None Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/plotting/ecdf/","title":"ecdf: Create an empirical cumulative distribution function plot"},{"location":"user_guide/plotting/ecdf/#ecdf-create-an-empirical-cumulative-distribution-function-plot","text":"A function to conveniently plot an empirical cumulative distribution function. from mlxtend.ecdf import ecdf","title":"ecdf: Create an empirical cumulative distribution function plot"},{"location":"user_guide/plotting/ecdf/#overview","text":"A function to conveniently plot an empirical cumulative distribution function (ECDF) and adding percentile thresholds for exploratory data analysis.","title":"Overview"},{"location":"user_guide/plotting/ecdf/#references","text":"-","title":"References"},{"location":"user_guide/plotting/ecdf/#example-1-ecdf","text":"from mlxtend.data import iris_data from mlxtend.plotting import ecdf import matplotlib.pyplot as plt X, y = iris_data() ax, _, _ = ecdf(x=X[:, 0], x_label='sepal length (cm)') plt.show()","title":"Example 1 - ECDF"},{"location":"user_guide/plotting/ecdf/#example-2-multiple-ecdfs","text":"from mlxtend.data import iris_data from mlxtend.plotting import ecdf import matplotlib.pyplot as plt X, y = iris_data() # first ecdf x1 = X[:, 0] ax, _, _ = ecdf(x1, x_label='cm') # second ecdf x2 = X[:, 1] ax, _, _ = ecdf(x2, ax=ax) plt.legend(['sepal length', 'sepal width']) plt.show()","title":"Example 2 - Multiple ECDFs"},{"location":"user_guide/plotting/ecdf/#example-3-ecdf-with-percentile-thresholds","text":"from mlxtend.data import iris_data from mlxtend.plotting import ecdf import matplotlib.pyplot as plt X, y = iris_data() ax, threshold, count = ecdf(x=X[:, 0], x_label='sepal length (cm)', percentile=0.8) plt.show() print('Feature threshold at the 80th percentile:', threshold) print('Number of samples below the threshold:', count) Feature threshold at the 80th percentile: 6.5 Number of samples below the threshold: 120","title":"Example 3 - ECDF with Percentile Thresholds"},{"location":"user_guide/plotting/ecdf/#api","text":"ecdf(x, y_label='ECDF', x_label=None, ax=None, percentile=None, ecdf_color=None, ecdf_marker='o', percentile_color='black', percentile_linestyle='--') Plots an Empirical Cumulative Distribution Function Parameters x : array or list, shape=[n_samples,] Array-like object containing the feature values y_label : str (default='ECDF') Text label for the y-axis x_label : str (default=None) Text label for the x-axis ax : matplotlib.axes.Axes (default: None) An existing matplotlib Axes. Creates one if ax=None percentile : float (default=None) Float between 0 and 1 for plotting a percentile threshold line ecdf_color : matplotlib color (default=None) Color for the ECDF plot; uses matplotlib defaults if None ecdf_marker : matplotlib marker (default='o') Marker style for the ECDF plot percentile_color : matplotlib color (default='black') Color for the percentile threshold if percentile is not None percentile_linestyle : matplotlib linestyle (default='--') Line style for the percentile threshold if percentile is not None Returns ax : matplotlib.axes.Axes object percentile_threshold : float Feature threshold at the percentile or None if percentile=None percentile_count : Number of if percentile is not None Number of samples that have a feature less or equal than the feature threshold at a percentile threshold or None if percentile=None Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/plotting/ecdf/","title":"API"},{"location":"user_guide/plotting/enrichment_plot/","text":"enrichment_plot: create an enrichment plot for cumulative counts A function to plot step plots of cumulative counts. from mlxtend.general import enrichment_plot Overview In enrichment plots, the y-axis can be interpreted as \"how many samples are less or equal to the corresponding x-axis label.\" References - Example 1 - Enrichment Plots from Pandas DataFrames import pandas as pd s1 = [1.1, 1.5] s2 = [2.1, 1.8] s3 = [3.1, 2.1] s4 = [3.9, 2.5] data = [s1, s2, s3, s4] df = pd.DataFrame(data, columns=['X1', 'X2']) df X1 X2 0 1.1 1.5 1 2.1 1.8 2 3.1 2.1 3 3.9 2.5 Plotting the data where the categories are determined by the unique values in the label column label_col . The x and y values are simply the column names of the DataFrame that we want to plot. import matplotlib.pyplot as plt from mlxtend.plotting import enrichment_plot ax = enrichment_plot(df, legend_loc='upper left') API enrichment_plot(df, colors='bgrkcy', markers=' ', linestyles='-', alpha=0.5, lw=2, where='post', grid=True, count_label='Count', xlim='auto', ylim='auto', invert_axes=False, legend_loc='best', ax=None) Plot stacked barplots Parameters df : pandas.DataFrame A pandas DataFrame where columns represent the different categories. colors: str (default: 'bgrcky') The colors of the bars. markers : str (default: ' ') Matplotlib markerstyles, e.g, 'sov' for square,circle, and triangle markers. linestyles : str (default: '-') Matplotlib linestyles, e.g., '-,--' to cycle normal and dashed lines. Note that the different linestyles need to be separated by commas. alpha : float (default: 0.5) Transparency level from 0.0 to 1.0. lw : int or float (default: 2) Linewidth parameter. where : {'post', 'pre', 'mid'} (default: 'post') Starting location of the steps. grid : bool (default: True ) Plots a grid if True. count_label : str (default: 'Count') Label for the \"Count\"-axis. xlim : 'auto' or array-like [min, max] (default: 'auto') Min and maximum position of the x-axis range. ylim : 'auto' or array-like [min, max] (default: 'auto') Min and maximum position of the y-axis range. invert_axes : bool (default: False) Plots count on the x-axis if True. legend_loc : str (default: 'best') Location of the plot legend {best, upper left, upper right, lower left, lower right} No legend if legend_loc=False ax : matplotlib axis, optional (default: None) Use this axis for plotting or make a new one otherwise Returns ax : matplotlib axis Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/plotting/enrichment_plot/","title":"enrichment_plot: create an enrichment plot for cumulative counts"},{"location":"user_guide/plotting/enrichment_plot/#enrichment_plot-create-an-enrichment-plot-for-cumulative-counts","text":"A function to plot step plots of cumulative counts. from mlxtend.general import enrichment_plot","title":"enrichment_plot: create an enrichment plot for cumulative counts"},{"location":"user_guide/plotting/enrichment_plot/#overview","text":"In enrichment plots, the y-axis can be interpreted as \"how many samples are less or equal to the corresponding x-axis label.\"","title":"Overview"},{"location":"user_guide/plotting/enrichment_plot/#references","text":"-","title":"References"},{"location":"user_guide/plotting/enrichment_plot/#example-1-enrichment-plots-from-pandas-dataframes","text":"import pandas as pd s1 = [1.1, 1.5] s2 = [2.1, 1.8] s3 = [3.1, 2.1] s4 = [3.9, 2.5] data = [s1, s2, s3, s4] df = pd.DataFrame(data, columns=['X1', 'X2']) df X1 X2 0 1.1 1.5 1 2.1 1.8 2 3.1 2.1 3 3.9 2.5 Plotting the data where the categories are determined by the unique values in the label column label_col . The x and y values are simply the column names of the DataFrame that we want to plot. import matplotlib.pyplot as plt from mlxtend.plotting import enrichment_plot ax = enrichment_plot(df, legend_loc='upper left')","title":"Example 1 - Enrichment Plots from Pandas DataFrames"},{"location":"user_guide/plotting/enrichment_plot/#api","text":"enrichment_plot(df, colors='bgrkcy', markers=' ', linestyles='-', alpha=0.5, lw=2, where='post', grid=True, count_label='Count', xlim='auto', ylim='auto', invert_axes=False, legend_loc='best', ax=None) Plot stacked barplots Parameters df : pandas.DataFrame A pandas DataFrame where columns represent the different categories. colors: str (default: 'bgrcky') The colors of the bars. markers : str (default: ' ') Matplotlib markerstyles, e.g, 'sov' for square,circle, and triangle markers. linestyles : str (default: '-') Matplotlib linestyles, e.g., '-,--' to cycle normal and dashed lines. Note that the different linestyles need to be separated by commas. alpha : float (default: 0.5) Transparency level from 0.0 to 1.0. lw : int or float (default: 2) Linewidth parameter. where : {'post', 'pre', 'mid'} (default: 'post') Starting location of the steps. grid : bool (default: True ) Plots a grid if True. count_label : str (default: 'Count') Label for the \"Count\"-axis. xlim : 'auto' or array-like [min, max] (default: 'auto') Min and maximum position of the x-axis range. ylim : 'auto' or array-like [min, max] (default: 'auto') Min and maximum position of the y-axis range. invert_axes : bool (default: False) Plots count on the x-axis if True. legend_loc : str (default: 'best') Location of the plot legend {best, upper left, upper right, lower left, lower right} No legend if legend_loc=False ax : matplotlib axis, optional (default: None) Use this axis for plotting or make a new one otherwise Returns ax : matplotlib axis Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/plotting/enrichment_plot/","title":"API"},{"location":"user_guide/plotting/heatmap/","text":"heatmap: Create a heatmap in matplotlib Utility function for creating a heatmap via matplotlib from mlxtend.plotting import heatmap Overview A simple function that creates nice-looking heatmaps from NumPy arrays using matplotlib and the Viridis color palette by default. References - Example 1 -- Simple heatmap from mlxtend.plotting import heatmap import matplotlib.pyplot as plt import numpy as np import pandas as pd np.random.seed(123) some_array = np.random.random((20, 30)) heatmap(some_array, figsize=(20, 10)) plt.show() The cell values can be suppressed by setting cell_values=False : heatmap(some_array, figsize=(20, 10), cell_values=False) plt.show() Example 2 -- Correlation matrix as heatmap from mlxtend.plotting import heatmap import matplotlib.pyplot as plt import numpy as np import pandas as pd df = pd.read_csv('https://raw.githubusercontent.com/rasbt/' 'python-machine-learning-book-2nd-edition' '/master/code/ch10/housing.data.txt', header=None, sep='\\s+') df.columns = ['CRIM', 'ZN', 'INDUS', 'CHAS', 'NOX', 'RM', 'AGE', 'DIS', 'RAD', 'TAX', 'PTRATIO', 'B', 'LSTAT', 'MEDV'] df.head() .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } CRIM ZN INDUS CHAS NOX RM AGE DIS RAD TAX PTRATIO B LSTAT MEDV 0 0.00632 18.0 2.31 0 0.538 6.575 65.2 4.0900 1 296.0 15.3 396.90 4.98 24.0 1 0.02731 0.0 7.07 0 0.469 6.421 78.9 4.9671 2 242.0 17.8 396.90 9.14 21.6 2 0.02729 0.0 7.07 0 0.469 7.185 61.1 4.9671 2 242.0 17.8 392.83 4.03 34.7 3 0.03237 0.0 2.18 0 0.458 6.998 45.8 6.0622 3 222.0 18.7 394.63 2.94 33.4 4 0.06905 0.0 2.18 0 0.458 7.147 54.2 6.0622 3 222.0 18.7 396.90 5.33 36.2 from matplotlib import cm cols = ['LSTAT', 'INDUS', 'NOX', 'RM', 'MEDV'] corrmat = np.corrcoef(df[cols].values.T) fig, ax = heatmap(corrmat, column_names=cols, row_names=cols, cmap=cm.PiYG) # set colorbar cutoff at -1, 1 for im in ax.get_images(): im.set_clim(-1, 1) plt.show() API heatmap(matrix, hide_spines=False, hide_ticks=False, figsize=None, cmap=None, colorbar=True, row_names=None, column_names=None, column_name_rotation=45, cell_values=True, cell_fmt='.2f', cell_font_size=None, text_color_threshold=None) Plot a heatmap via matplotlib. Parameters conf_mat : array-like, shape = [n_rows, n_columns] And arbitrary 2D array. hide_spines : bool (default: False) Hides axis spines if True. hide_ticks : bool (default: False) Hides axis ticks if True figsize : tuple (default: (2.5, 2.5)) Height and width of the figure cmap : matplotlib colormap (default: None ) Uses matplotlib.pyplot.cm.viridis if None colorbar : bool (default: True) Shows a colorbar if True row_names : array-like, shape = [n_rows] (default: None) List of row names to be used as y-axis tick labels. column_names : array-like, shape = [n_columns] (default: None) List of column names to be used as x-axis tick labels. column_name_rotation : int (default: 45) Number of degrees for rotating column x-tick labels. cell_values : bool (default: True) Plots cell values if True. cell_fmt : string (default: '.2f') Format specification for cell values (if cell_values=True ) cell_font_size : int (default: None) Font size for cell values (if cell_values=True ) text_color_threshold : float (default: None) Threshold for the black/white text threshold of the text annotation. Default (None) tried to infer a good threshold automatically using np.max(normed_matrix) / 2 . Returns fig, ax : matplotlib.pyplot subplot objects Figure and axis elements of the subplot. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/plotting/heatmap/","title":"heatmap: Create a heatmap in matplotlib"},{"location":"user_guide/plotting/heatmap/#heatmap-create-a-heatmap-in-matplotlib","text":"Utility function for creating a heatmap via matplotlib from mlxtend.plotting import heatmap","title":"heatmap: Create a heatmap in matplotlib"},{"location":"user_guide/plotting/heatmap/#overview","text":"A simple function that creates nice-looking heatmaps from NumPy arrays using matplotlib and the Viridis color palette by default.","title":"Overview"},{"location":"user_guide/plotting/heatmap/#references","text":"-","title":"References"},{"location":"user_guide/plotting/heatmap/#example-1-simple-heatmap","text":"from mlxtend.plotting import heatmap import matplotlib.pyplot as plt import numpy as np import pandas as pd np.random.seed(123) some_array = np.random.random((20, 30)) heatmap(some_array, figsize=(20, 10)) plt.show() The cell values can be suppressed by setting cell_values=False : heatmap(some_array, figsize=(20, 10), cell_values=False) plt.show()","title":"Example 1 -- Simple heatmap"},{"location":"user_guide/plotting/heatmap/#example-2-correlation-matrix-as-heatmap","text":"from mlxtend.plotting import heatmap import matplotlib.pyplot as plt import numpy as np import pandas as pd df = pd.read_csv('https://raw.githubusercontent.com/rasbt/' 'python-machine-learning-book-2nd-edition' '/master/code/ch10/housing.data.txt', header=None, sep='\\s+') df.columns = ['CRIM', 'ZN', 'INDUS', 'CHAS', 'NOX', 'RM', 'AGE', 'DIS', 'RAD', 'TAX', 'PTRATIO', 'B', 'LSTAT', 'MEDV'] df.head() .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } CRIM ZN INDUS CHAS NOX RM AGE DIS RAD TAX PTRATIO B LSTAT MEDV 0 0.00632 18.0 2.31 0 0.538 6.575 65.2 4.0900 1 296.0 15.3 396.90 4.98 24.0 1 0.02731 0.0 7.07 0 0.469 6.421 78.9 4.9671 2 242.0 17.8 396.90 9.14 21.6 2 0.02729 0.0 7.07 0 0.469 7.185 61.1 4.9671 2 242.0 17.8 392.83 4.03 34.7 3 0.03237 0.0 2.18 0 0.458 6.998 45.8 6.0622 3 222.0 18.7 394.63 2.94 33.4 4 0.06905 0.0 2.18 0 0.458 7.147 54.2 6.0622 3 222.0 18.7 396.90 5.33 36.2 from matplotlib import cm cols = ['LSTAT', 'INDUS', 'NOX', 'RM', 'MEDV'] corrmat = np.corrcoef(df[cols].values.T) fig, ax = heatmap(corrmat, column_names=cols, row_names=cols, cmap=cm.PiYG) # set colorbar cutoff at -1, 1 for im in ax.get_images(): im.set_clim(-1, 1) plt.show()","title":"Example 2 -- Correlation matrix as heatmap"},{"location":"user_guide/plotting/heatmap/#api","text":"heatmap(matrix, hide_spines=False, hide_ticks=False, figsize=None, cmap=None, colorbar=True, row_names=None, column_names=None, column_name_rotation=45, cell_values=True, cell_fmt='.2f', cell_font_size=None, text_color_threshold=None) Plot a heatmap via matplotlib. Parameters conf_mat : array-like, shape = [n_rows, n_columns] And arbitrary 2D array. hide_spines : bool (default: False) Hides axis spines if True. hide_ticks : bool (default: False) Hides axis ticks if True figsize : tuple (default: (2.5, 2.5)) Height and width of the figure cmap : matplotlib colormap (default: None ) Uses matplotlib.pyplot.cm.viridis if None colorbar : bool (default: True) Shows a colorbar if True row_names : array-like, shape = [n_rows] (default: None) List of row names to be used as y-axis tick labels. column_names : array-like, shape = [n_columns] (default: None) List of column names to be used as x-axis tick labels. column_name_rotation : int (default: 45) Number of degrees for rotating column x-tick labels. cell_values : bool (default: True) Plots cell values if True. cell_fmt : string (default: '.2f') Format specification for cell values (if cell_values=True ) cell_font_size : int (default: None) Font size for cell values (if cell_values=True ) text_color_threshold : float (default: None) Threshold for the black/white text threshold of the text annotation. Default (None) tried to infer a good threshold automatically using np.max(normed_matrix) / 2 . Returns fig, ax : matplotlib.pyplot subplot objects Figure and axis elements of the subplot. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/plotting/heatmap/","title":"API"},{"location":"user_guide/plotting/plot_confusion_matrix/","text":"plot_confusion_matrix: Visualize confusion matrices Utility function for visualizing confusion matrices via matplotlib from mlxtend.plotting import plot_confusion_matrix Overview Confusion Matrix For more information on confusion matrices, please see mlxtend.evaluate.confusion_matrix . References - Example 1 - Binary from mlxtend.plotting import plot_confusion_matrix import matplotlib.pyplot as plt import numpy as np binary1 = np.array([[4, 1], [1, 2]]) fig, ax = plot_confusion_matrix(conf_mat=binary1) plt.show() binary2 = np.array([[21, 1], [3, 1]]) fig, ax = plot_confusion_matrix(conf_mat=binary2, figsize=(2, 2)) plt.show() Example 2 - Binary absolute and relative with colorbar binary = np.array([[4, 1], [1, 2]]) fig, ax = plot_confusion_matrix(conf_mat=binary, show_absolute=True, show_normed=True, colorbar=True) plt.show() Example 3 - Multiclass relative multiclass = np.array([[2, 1, 0, 0], [1, 2, 0, 0], [0, 0, 1, 0], [0, 0, 0, 1]]) fig, ax = plot_confusion_matrix(conf_mat=multiclass, colorbar=True, show_absolute=False, show_normed=True) plt.show() Example 4 - Add Class Names multiclass = np.array([[2, 1, 0, 0], [1, 2, 0, 0], [0, 0, 1, 0], [0, 0, 0, 1]]) class_names = ['class a', 'class b', 'class c', 'class d'] fig, ax = plot_confusion_matrix(conf_mat=multiclass, colorbar=True, show_absolute=False, show_normed=True, class_names=class_names) plt.show() Example 5 - Changing Color Maps and Font Color Matplotlib color maps can be chosen as alternative color map via the cmap argument. A list of colormaps can be found here: https://matplotlib.org/stable/tutorials/colors/colormaps.html multiclass = np.array([[2, 1, 0, 0], [1, 2, 0, 0], [0, 0, 1, 0], [0, 0, 0, 1]]) fig, ax = plot_confusion_matrix(conf_mat=multiclass, colorbar=True, cmap='summer') plt.show() As shown above, the font color threshold may not work for certain color maps. By default all values larger than 0.5 times the maximum cell value are converted to white, and everything equal or smaller than 0.5 times the maximum cell value are converted to black. If you want to change all values above to e.g., white, you can set the color threshold to a negative number. Or, if you want to make all the font colors black, choose ta threshold equal to or greater than 1. fig, ax = plot_confusion_matrix(conf_mat=multiclass, colorbar=True, fontcolor_threshold=1, cmap='summer') plt.show() Example 6 - Normalizing Colormaps to Highlight Off-Diagonals Suppose we have the following confusion matrix for a high-accuracy classifier: class_dict = {0: 'airplane', 1: 'automobile', 2: 'bird', 3: 'cat', 4: 'deer', 5: 'dog', 6: 'frog'} cmat = np.array([[972, 0, 1, 1, 1, 1, 3], [0, 1123, 3, 1, 0, 1, 2], [2, 0, 1025, 0, 0, 0, 1], [0, 0, 0, 1005, 0, 2, 0], [0, 1, 1, 0, 967, 0, 4], [0, 0, 0, 6, 0, 881, 3], [2, 3, 0, 1, 3, 4, 941]]) fig, ax = plot_confusion_matrix( conf_mat=cmat, class_names=class_dict.values(), ) It can be hard to notice the cells where the models makes mistakes. With a log-normalized colormap, these mistakes off the diagonal become easier to see at a glace: import matplotlib fig, ax = plot_confusion_matrix( conf_mat=cmat, class_names=class_dict.values(), norm_colormap=matplotlib.colors.LogNorm() ) API plot_confusion_matrix(conf_mat, hide_spines=False, hide_ticks=False, figsize=None, cmap=None, colorbar=False, show_absolute=True, show_normed=False, class_names=None, figure=None, axis=None, fontcolor_threshold=0.5) Plot a confusion matrix via matplotlib. Parameters conf_mat : array-like, shape = [n_classes, n_classes] Confusion matrix from evaluate.confusion matrix. hide_spines : bool (default: False) Hides axis spines if True. hide_ticks : bool (default: False) Hides axis ticks if True figsize : tuple (default: (2.5, 2.5)) Height and width of the figure cmap : matplotlib colormap (default: None ) Uses matplotlib.pyplot.cm.Blues if None colorbar : bool (default: False) Shows a colorbar if True show_absolute : bool (default: True) Shows absolute confusion matrix coefficients if True. At least one of show_absolute or show_normed must be True. show_normed : bool (default: False) Shows normed confusion matrix coefficients if True. The normed confusion matrix coefficients give the proportion of training examples per class that are assigned the correct label. At least one of show_absolute or show_normed must be True. class_names : array-like, shape = [n_classes] (default: None) List of class names. If not None , ticks will be set to these values. figure : None or Matplotlib figure (default: None) If None will create a new figure. axis : None or Matplotlib figure axis (default: None) If None will create a new axis. fontcolor_threshold : Float (default: 0.5) Sets a threshold for choosing black and white font colors for the cells. By default all values larger than 0.5 times the maximum cell value are converted to white, and everything equal or smaller than 0.5 times the maximum cell value are converted to black. Returns fig, ax : matplotlib.pyplot subplot objects Figure and axis elements of the subplot. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/plotting/plot_confusion_matrix/","title":"plot_confusion_matrix: Visualize confusion matrices"},{"location":"user_guide/plotting/plot_confusion_matrix/#plot_confusion_matrix-visualize-confusion-matrices","text":"Utility function for visualizing confusion matrices via matplotlib from mlxtend.plotting import plot_confusion_matrix","title":"plot_confusion_matrix: Visualize confusion matrices"},{"location":"user_guide/plotting/plot_confusion_matrix/#overview","text":"","title":"Overview"},{"location":"user_guide/plotting/plot_confusion_matrix/#confusion-matrix","text":"For more information on confusion matrices, please see mlxtend.evaluate.confusion_matrix .","title":"Confusion Matrix"},{"location":"user_guide/plotting/plot_confusion_matrix/#references","text":"-","title":"References"},{"location":"user_guide/plotting/plot_confusion_matrix/#example-1-binary","text":"from mlxtend.plotting import plot_confusion_matrix import matplotlib.pyplot as plt import numpy as np binary1 = np.array([[4, 1], [1, 2]]) fig, ax = plot_confusion_matrix(conf_mat=binary1) plt.show() binary2 = np.array([[21, 1], [3, 1]]) fig, ax = plot_confusion_matrix(conf_mat=binary2, figsize=(2, 2)) plt.show()","title":"Example 1 - Binary"},{"location":"user_guide/plotting/plot_confusion_matrix/#example-2-binary-absolute-and-relative-with-colorbar","text":"binary = np.array([[4, 1], [1, 2]]) fig, ax = plot_confusion_matrix(conf_mat=binary, show_absolute=True, show_normed=True, colorbar=True) plt.show()","title":"Example 2 - Binary absolute and relative with colorbar"},{"location":"user_guide/plotting/plot_confusion_matrix/#example-3-multiclass-relative","text":"multiclass = np.array([[2, 1, 0, 0], [1, 2, 0, 0], [0, 0, 1, 0], [0, 0, 0, 1]]) fig, ax = plot_confusion_matrix(conf_mat=multiclass, colorbar=True, show_absolute=False, show_normed=True) plt.show()","title":"Example 3 - Multiclass relative"},{"location":"user_guide/plotting/plot_confusion_matrix/#example-4-add-class-names","text":"multiclass = np.array([[2, 1, 0, 0], [1, 2, 0, 0], [0, 0, 1, 0], [0, 0, 0, 1]]) class_names = ['class a', 'class b', 'class c', 'class d'] fig, ax = plot_confusion_matrix(conf_mat=multiclass, colorbar=True, show_absolute=False, show_normed=True, class_names=class_names) plt.show()","title":"Example 4 - Add Class Names"},{"location":"user_guide/plotting/plot_confusion_matrix/#example-5-changing-color-maps-and-font-color","text":"Matplotlib color maps can be chosen as alternative color map via the cmap argument. A list of colormaps can be found here: https://matplotlib.org/stable/tutorials/colors/colormaps.html multiclass = np.array([[2, 1, 0, 0], [1, 2, 0, 0], [0, 0, 1, 0], [0, 0, 0, 1]]) fig, ax = plot_confusion_matrix(conf_mat=multiclass, colorbar=True, cmap='summer') plt.show() As shown above, the font color threshold may not work for certain color maps. By default all values larger than 0.5 times the maximum cell value are converted to white, and everything equal or smaller than 0.5 times the maximum cell value are converted to black. If you want to change all values above to e.g., white, you can set the color threshold to a negative number. Or, if you want to make all the font colors black, choose ta threshold equal to or greater than 1. fig, ax = plot_confusion_matrix(conf_mat=multiclass, colorbar=True, fontcolor_threshold=1, cmap='summer') plt.show()","title":"Example 5 - Changing Color Maps and Font Color"},{"location":"user_guide/plotting/plot_confusion_matrix/#example-6-normalizing-colormaps-to-highlight-off-diagonals","text":"Suppose we have the following confusion matrix for a high-accuracy classifier: class_dict = {0: 'airplane', 1: 'automobile', 2: 'bird', 3: 'cat', 4: 'deer', 5: 'dog', 6: 'frog'} cmat = np.array([[972, 0, 1, 1, 1, 1, 3], [0, 1123, 3, 1, 0, 1, 2], [2, 0, 1025, 0, 0, 0, 1], [0, 0, 0, 1005, 0, 2, 0], [0, 1, 1, 0, 967, 0, 4], [0, 0, 0, 6, 0, 881, 3], [2, 3, 0, 1, 3, 4, 941]]) fig, ax = plot_confusion_matrix( conf_mat=cmat, class_names=class_dict.values(), ) It can be hard to notice the cells where the models makes mistakes. With a log-normalized colormap, these mistakes off the diagonal become easier to see at a glace: import matplotlib fig, ax = plot_confusion_matrix( conf_mat=cmat, class_names=class_dict.values(), norm_colormap=matplotlib.colors.LogNorm() )","title":"Example 6 - Normalizing Colormaps to Highlight Off-Diagonals"},{"location":"user_guide/plotting/plot_confusion_matrix/#api","text":"plot_confusion_matrix(conf_mat, hide_spines=False, hide_ticks=False, figsize=None, cmap=None, colorbar=False, show_absolute=True, show_normed=False, class_names=None, figure=None, axis=None, fontcolor_threshold=0.5) Plot a confusion matrix via matplotlib. Parameters conf_mat : array-like, shape = [n_classes, n_classes] Confusion matrix from evaluate.confusion matrix. hide_spines : bool (default: False) Hides axis spines if True. hide_ticks : bool (default: False) Hides axis ticks if True figsize : tuple (default: (2.5, 2.5)) Height and width of the figure cmap : matplotlib colormap (default: None ) Uses matplotlib.pyplot.cm.Blues if None colorbar : bool (default: False) Shows a colorbar if True show_absolute : bool (default: True) Shows absolute confusion matrix coefficients if True. At least one of show_absolute or show_normed must be True. show_normed : bool (default: False) Shows normed confusion matrix coefficients if True. The normed confusion matrix coefficients give the proportion of training examples per class that are assigned the correct label. At least one of show_absolute or show_normed must be True. class_names : array-like, shape = [n_classes] (default: None) List of class names. If not None , ticks will be set to these values. figure : None or Matplotlib figure (default: None) If None will create a new figure. axis : None or Matplotlib figure axis (default: None) If None will create a new axis. fontcolor_threshold : Float (default: 0.5) Sets a threshold for choosing black and white font colors for the cells. By default all values larger than 0.5 times the maximum cell value are converted to white, and everything equal or smaller than 0.5 times the maximum cell value are converted to black. Returns fig, ax : matplotlib.pyplot subplot objects Figure and axis elements of the subplot. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/plotting/plot_confusion_matrix/","title":"API"},{"location":"user_guide/plotting/plot_decision_regions/","text":"plot_decision_regions: Visualize the decision regions of a classifier A function for plotting decision regions of classifiers in 1 or 2 dimensions. from mlxtend.plotting import plot_decision_regions References Example 1 - Decision regions in 2D from mlxtend.plotting import plot_decision_regions import matplotlib.pyplot as plt from sklearn import datasets from sklearn.svm import SVC # Loading some example data iris = datasets.load_iris() X = iris.data[:, [0, 2]] y = iris.target # Training a classifier svm = SVC(C=0.5, kernel='linear') svm.fit(X, y) # Plotting decision regions plot_decision_regions(X, y, clf=svm, legend=2) # Adding axes annotations plt.xlabel('sepal length [cm]') plt.ylabel('petal length [cm]') plt.title('SVM on Iris') plt.show() Example 2 - Decision regions in 1D from mlxtend.plotting import plot_decision_regions import matplotlib.pyplot as plt from sklearn import datasets from sklearn.svm import SVC # Loading some example data iris = datasets.load_iris() X = iris.data[:, 2] X = X[:, None] y = iris.target # Training a classifier svm = SVC(C=0.5, kernel='linear') svm.fit(X, y) # Plotting decision regions plot_decision_regions(X, y, clf=svm, legend=2) # Adding axes annotations plt.xlabel('sepal length [cm]') plt.title('SVM on Iris') plt.show() Example 3 - Decision Region Grids from sklearn.linear_model import LogisticRegression from sklearn.naive_bayes import GaussianNB from sklearn.ensemble import RandomForestClassifier from sklearn.svm import SVC from sklearn import datasets import numpy as np # Initializing Classifiers clf1 = LogisticRegression(random_state=1, solver='newton-cg', multi_class='multinomial') clf2 = RandomForestClassifier(random_state=1, n_estimators=100) clf3 = GaussianNB() clf4 = SVC(gamma='auto') # Loading some example data iris = datasets.load_iris() X = iris.data[:, [0,2]] y = iris.target import matplotlib.pyplot as plt from mlxtend.plotting import plot_decision_regions import matplotlib.gridspec as gridspec import itertools gs = gridspec.GridSpec(2, 2) fig = plt.figure(figsize=(10,8)) labels = ['Logistic Regression', 'Random Forest', 'Naive Bayes', 'SVM'] for clf, lab, grd in zip([clf1, clf2, clf3, clf4], labels, itertools.product([0, 1], repeat=2)): clf.fit(X, y) ax = plt.subplot(gs[grd[0], grd[1]]) fig = plot_decision_regions(X=X, y=y, clf=clf, legend=2) plt.title(lab) plt.show() Example 4 - Highlighting Test Data Points from mlxtend.plotting import plot_decision_regions from mlxtend.preprocessing import shuffle_arrays_unison import matplotlib.pyplot as plt from sklearn import datasets from sklearn.svm import SVC # Loading some example data iris = datasets.load_iris() X, y = iris.data[:, [0,2]], iris.target X, y = shuffle_arrays_unison(arrays=[X, y], random_seed=3) X_train, y_train = X[:100], y[:100] X_test, y_test = X[100:], y[100:] # Training a classifier svm = SVC(C=0.5, kernel='linear') svm.fit(X_train, y_train) # Plotting decision regions plot_decision_regions(X, y, clf=svm, legend=2, X_highlight=X_test) # Adding axes annotations plt.xlabel('sepal length [cm]') plt.ylabel('petal length [cm]') plt.title('SVM on Iris') plt.show() Example 5 - Evaluating Classifier Behavior on Non-Linear Problems from sklearn.linear_model import LogisticRegression from sklearn.naive_bayes import GaussianNB from sklearn.ensemble import RandomForestClassifier from sklearn.svm import SVC # Initializing Classifiers clf1 = LogisticRegression(random_state=1, solver='lbfgs') clf2 = RandomForestClassifier(n_estimators=100, random_state=1) clf3 = GaussianNB() clf4 = SVC(gamma='auto') # Loading Plotting Utilities import matplotlib.pyplot as plt import matplotlib.gridspec as gridspec import itertools from mlxtend.plotting import plot_decision_regions import numpy as np XOR xx, yy = np.meshgrid(np.linspace(-3, 3, 50), np.linspace(-3, 3, 50)) rng = np.random.RandomState(0) X = rng.randn(300, 2) y = np.array(np.logical_xor(X[:, 0] > 0, X[:, 1] > 0), dtype=int) gs = gridspec.GridSpec(2, 2) fig = plt.figure(figsize=(10,8)) labels = ['Logistic Regression', 'Random Forest', 'Naive Bayes', 'SVM'] for clf, lab, grd in zip([clf1, clf2, clf3, clf4], labels, itertools.product([0, 1], repeat=2)): clf.fit(X, y) ax = plt.subplot(gs[grd[0], grd[1]]) fig = plot_decision_regions(X=X, y=y, clf=clf, legend=2) plt.title(lab) plt.show() Half-Moons from sklearn.datasets import make_moons X, y = make_moons(n_samples=100, random_state=123) gs = gridspec.GridSpec(2, 2) fig = plt.figure(figsize=(10,8)) labels = ['Logistic Regression', 'Random Forest', 'Naive Bayes', 'SVM'] for clf, lab, grd in zip([clf1, clf2, clf3, clf4], labels, itertools.product([0, 1], repeat=2)): clf.fit(X, y) ax = plt.subplot(gs[grd[0], grd[1]]) fig = plot_decision_regions(X=X, y=y, clf=clf, legend=2) plt.title(lab) plt.show() Concentric Circles from sklearn.datasets import make_circles X, y = make_circles(n_samples=1000, random_state=123, noise=0.1, factor=0.2) gs = gridspec.GridSpec(2, 2) fig = plt.figure(figsize=(10,8)) labels = ['Logistic Regression', 'Random Forest', 'Naive Bayes', 'SVM'] for clf, lab, grd in zip([clf1, clf2, clf3, clf4], labels, itertools.product([0, 1], repeat=2)): clf.fit(X, y) ax = plt.subplot(gs[grd[0], grd[1]]) fig = plot_decision_regions(X=X, y=y, clf=clf, legend=2) plt.title(lab) plt.show() Example 6 - Working with existing axes objects using subplots import matplotlib.pyplot as plt from mlxtend.plotting import plot_decision_regions from sklearn.linear_model import LogisticRegression from sklearn.naive_bayes import GaussianNB from sklearn import datasets import numpy as np # Loading some example data iris = datasets.load_iris() X = iris.data[:, 2] X = X[:, None] y = iris.target # Initializing and fitting classifiers clf1 = LogisticRegression(random_state=1, solver='lbfgs', multi_class='multinomial') clf2 = GaussianNB() clf1.fit(X, y) clf2.fit(X, y) fig, axes = plt.subplots(1, 2, figsize=(10, 3)) fig = plot_decision_regions(X=X, y=y, clf=clf1, ax=axes[0], legend=2) fig = plot_decision_regions(X=X, y=y, clf=clf2, ax=axes[1], legend=1) plt.show() Example 7 - Decision regions with more than two training features from mlxtend.plotting import plot_decision_regions import matplotlib.pyplot as plt from sklearn import datasets from sklearn.svm import SVC # Loading some example data X, y = datasets.make_blobs(n_samples=600, n_features=3, centers=[[2, 2, -2],[-2, -2, 2]], cluster_std=[2, 2], random_state=2) # Training a classifier svm = SVC(gamma='auto') svm.fit(X, y) # Plotting decision regions fig, ax = plt.subplots() # Decision region for feature 3 = 1.5 value = 1.5 # Plot training sample with feature 3 = 1.5 +/- 0.75 width = 0.75 plot_decision_regions(X, y, clf=svm, filler_feature_values={2: value}, filler_feature_ranges={2: width}, legend=2, ax=ax) ax.set_xlabel('Feature 1') ax.set_ylabel('Feature 2') ax.set_title('Feature 3 = {}'.format(value)) # Adding axes annotations fig.suptitle('SVM on make_blobs') plt.show() Example 8 - Grid of decision region slices from mlxtend.plotting import plot_decision_regions import matplotlib.pyplot as plt from sklearn import datasets from sklearn.svm import SVC # Loading some example data X, y = datasets.make_blobs(n_samples=500, n_features=3, centers=[[2, 2, -2],[-2, -2, 2]], cluster_std=[2, 2], random_state=2) # Training a classifier svm = SVC(gamma='auto') svm.fit(X, y) # Plotting decision regions fig, axarr = plt.subplots(2, 2, figsize=(10,8), sharex=True, sharey=True) values = [-4.0, -1.0, 1.0, 4.0] width = 0.75 for value, ax in zip(values, axarr.flat): plot_decision_regions(X, y, clf=svm, filler_feature_values={2: value}, filler_feature_ranges={2: width}, legend=2, ax=ax) ax.set_xlabel('Feature 1') ax.set_ylabel('Feature 2') ax.set_title('Feature 3 = {}'.format(value)) # Adding axes annotations fig.suptitle('SVM on make_blobs') plt.show() Example 9 - Customizing the plotting style from mlxtend.plotting import plot_decision_regions from mlxtend.preprocessing import shuffle_arrays_unison import matplotlib.pyplot as plt from sklearn import datasets from sklearn.svm import SVC # Loading some example data iris = datasets.load_iris() X = iris.data[:, [0, 2]] y = iris.target X, y = shuffle_arrays_unison(arrays=[X, y], random_seed=3) X_train, y_train = X[:100], y[:100] X_test, y_test = X[100:], y[100:] # Training a classifier svm = SVC(C=0.5, kernel='linear') svm.fit(X_train, y_train) # Specify keyword arguments to be passed to underlying plotting functions scatter_kwargs = {'s': 120, 'edgecolor': None, 'alpha': 0.7} contourf_kwargs = {'alpha': 0.2} scatter_highlight_kwargs = {'s': 120, 'label': 'Test data', 'alpha': 0.7} # Plotting decision regions plot_decision_regions(X, y, clf=svm, legend=2, X_highlight=X_test, scatter_kwargs=scatter_kwargs, contourf_kwargs=contourf_kwargs, scatter_highlight_kwargs=scatter_highlight_kwargs) # Adding axes annotations plt.xlabel('sepal length [cm]') plt.ylabel('petal length [cm]') plt.title('SVM on Iris') plt.show() Example 10 - Providing your own legend labels Custom legend labels can be provided by returning the axis object(s) from the plot_decision_region function and then getting the handles and labels of the legend. Custom handles (i.e., labels) can then be provided via ax.legend ax = plot_decision_regions(X, y, clf=svm, legend=0) handles, labels = ax.get_legend_handles_labels() ax.legend(handles, ['class 0', 'class 1', 'class 2'], framealpha=0.3, scatterpoints=1) An example is shown below. from mlxtend.plotting import plot_decision_regions import matplotlib.pyplot as plt from sklearn import datasets from sklearn.svm import SVC # Loading some example data iris = datasets.load_iris() X = iris.data[:, [0, 2]] y = iris.target # Training a classifier svm = SVC(C=0.5, kernel='linear') svm.fit(X, y) # Plotting decision regions ax = plot_decision_regions(X, y, clf=svm, legend=0) # Adding axes annotations plt.xlabel('sepal length [cm]') plt.ylabel('petal length [cm]') plt.title('SVM on Iris') handles, labels = ax.get_legend_handles_labels() ax.legend(handles, ['class square', 'class triangle', 'class circle'], framealpha=0.3, scatterpoints=1) plt.show() Example 11 - Plots with different zoom factors from mlxtend.plotting import plot_decision_regions import matplotlib.pyplot as plt from sklearn import datasets from sklearn.svm import SVC # Loading some example data iris = datasets.load_iris() X = iris.data[:, [0, 2]] y = iris.target # Training a classifier svm = SVC(C=0.5, kernel='linear') svm.fit(X, y) #sk-container-id-1 {color: black;background-color: white;}#sk-container-id-1 pre{padding: 0;}#sk-container-id-1 div.sk-toggleable {background-color: white;}#sk-container-id-1 label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.3em;box-sizing: border-box;text-align: center;}#sk-container-id-1 label.sk-toggleable__label-arrow:before {content: \"\u25b8\";float: left;margin-right: 0.25em;color: #696969;}#sk-container-id-1 label.sk-toggleable__label-arrow:hover:before {color: black;}#sk-container-id-1 div.sk-estimator:hover label.sk-toggleable__label-arrow:before {color: black;}#sk-container-id-1 div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}#sk-container-id-1 div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}#sk-container-id-1 input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}#sk-container-id-1 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {content: \"\u25be\";}#sk-container-id-1 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}#sk-container-id-1 div.sk-estimator {font-family: monospace;background-color: #f0f8ff;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;margin-bottom: 0.5em;}#sk-container-id-1 div.sk-estimator:hover {background-color: #d4ebff;}#sk-container-id-1 div.sk-parallel-item::after {content: \"\";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}#sk-container-id-1 div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 div.sk-serial::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: 0;}#sk-container-id-1 div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;padding-right: 0.2em;padding-left: 0.2em;position: relative;}#sk-container-id-1 div.sk-item {position: relative;z-index: 1;}#sk-container-id-1 div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;position: relative;}#sk-container-id-1 div.sk-item::before, #sk-container-id-1 div.sk-parallel-item::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: -1;}#sk-container-id-1 div.sk-parallel-item {display: flex;flex-direction: column;z-index: 1;position: relative;background-color: white;}#sk-container-id-1 div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}#sk-container-id-1 div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}#sk-container-id-1 div.sk-parallel-item:only-child::after {width: 0;}#sk-container-id-1 div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0 0.4em 0.5em 0.4em;box-sizing: border-box;padding-bottom: 0.4em;background-color: white;}#sk-container-id-1 div.sk-label label {font-family: monospace;font-weight: bold;display: inline-block;line-height: 1.2em;}#sk-container-id-1 div.sk-label-container {text-align: center;}#sk-container-id-1 div.sk-container {/* jupyter's `normalize.less` sets `[hidden] { display: none; }` but bootstrap.min.css set `[hidden] { display: none !important; }` so we also need the `!important` here to be able to override the default hidden behavior on the sphinx rendered scikit-learn.org. See: https://github.com/scikit-learn/scikit-learn/issues/21755 */display: inline-block !important;position: relative;}#sk-container-id-1 div.sk-text-repr-fallback {display: none;} SVC(C=0.5, kernel='linear') In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org. SVC SVC(C=0.5, kernel='linear') Default Zoom Factor plot_decision_regions(X, y, clf=svm, zoom_factor=1.) plt.show() Zooming out plot_decision_regions(X, y, clf=svm, zoom_factor=0.1) plt.show() Zooming in Note that while zooming in (by choosing a zoom_factor > 1.0) the plots are still created such that all data points are shown in the plot. plot_decision_regions(X, y, clf=svm, zoom_factor=2.0) plt.show() Cropping the axes In order to zoom in further, which means that some training examples won't be shown, you can simply crop the axes as shown below: plot_decision_regions(X, y, clf=svm, zoom_factor=2.0) plt.xlim(5, 6) plt.ylim(2, 5) plt.show() Example 12 - Using classifiers that expect onehot-encoded outputs (Keras) Most objects for classification that mimick the scikit-learn estimator API should be compatible with the plot_decision_regions function. However, if the classification model (e.g., a typical Keras model) output onehot-encoded predictions, we have to use an additional trick. I.e., for onehot encoded outputs, we need to wrap the Keras model into a class that converts these onehot encoded variables into integers. Such a wrapper class can be as simple as the following: class Onehot2Int(object): def __init__(self, model): self.model = model def predict(self, X): y_pred = self.model.predict(X) return np.argmax(y_pred, axis=1) The example below illustrates how the Onehot2Int class can be used with a Keras model that outputs onehot encoded labels: import keras from keras.models import Sequential from keras.layers import Dense import matplotlib.pyplot as plt import numpy as np from mlxtend.data import iris_data from mlxtend.preprocessing import standardize from mlxtend.plotting import plot_decision_regions from keras.utils import to_categorical X, y = iris_data() X = X[:, [2, 3]] X = standardize(X) # OneHot encoding y_onehot = to_categorical(y) # Create the model np.random.seed(123) model = Sequential() model.add(Dense(8, input_shape=(2,), activation='relu', kernel_initializer='he_uniform')) model.add(Dense(4, activation='relu', kernel_initializer='he_uniform')) model.add(Dense(3, activation='softmax')) # Configure the model and start training model.compile(loss=\"categorical_crossentropy\", optimizer=keras.optimizers.Adam(lr=0.005), metrics=['accuracy']) history = model.fit(X, y_onehot, epochs=10, batch_size=5, verbose=1, validation_split=0.1) Epoch 1/10 1/27 [>.............................] - ETA: 3s - loss: 1.2769 - accuracy: 0.4000 /Users/sebastianraschka/miniforge3/envs/mlxtend/lib/python3.8/site-packages/keras/optimizers/optimizer_v2/adam.py:117: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead. super().__init__(name, **kwargs) 2023-03-28 17:48:13.901264: W tensorflow/tsl/platform/profile_utils/cpu_utils.cc:128] Failed to get CPU frequency: 0 Hz 27/27 [==============================] - 0s 3ms/step - loss: 0.9526 - accuracy: 0.4222 - val_loss: 1.2656 - val_accuracy: 0.0000e+00 Epoch 2/10 27/27 [==============================] - 0s 834us/step - loss: 0.7062 - accuracy: 0.6741 - val_loss: 1.0939 - val_accuracy: 0.0000e+00 Epoch 3/10 27/27 [==============================] - 0s 808us/step - loss: 0.6461 - accuracy: 0.7111 - val_loss: 1.0705 - val_accuracy: 0.0667 Epoch 4/10 27/27 [==============================] - 0s 767us/step - loss: 0.6145 - accuracy: 0.7185 - val_loss: 1.0518 - val_accuracy: 0.0000e+00 Epoch 5/10 27/27 [==============================] - 0s 746us/step - loss: 0.5877 - accuracy: 0.7185 - val_loss: 1.0470 - val_accuracy: 0.0000e+00 Epoch 6/10 27/27 [==============================] - 0s 740us/step - loss: 0.5496 - accuracy: 0.7333 - val_loss: 1.0275 - val_accuracy: 0.0000e+00 Epoch 7/10 27/27 [==============================] - 0s 734us/step - loss: 0.4985 - accuracy: 0.7333 - val_loss: 1.0131 - val_accuracy: 0.0000e+00 Epoch 8/10 27/27 [==============================] - 0s 739us/step - loss: 0.4365 - accuracy: 0.7333 - val_loss: 0.9634 - val_accuracy: 0.0000e+00 Epoch 9/10 27/27 [==============================] - 0s 729us/step - loss: 0.3875 - accuracy: 0.7333 - val_loss: 0.9442 - val_accuracy: 0.0000e+00 Epoch 10/10 27/27 [==============================] - 0s 764us/step - loss: 0.3402 - accuracy: 0.7407 - val_loss: 0.8565 - val_accuracy: 0.0000e+00 # Wrap keras model model_no_ohe = Onehot2Int(model) # Plot decision boundary plot_decision_regions(X, y, clf=model_no_ohe) plt.show() 9600/9600 [==============================] - 3s 289us/step API plot_decision_regions(X, y, clf, feature_index=None, filler_feature_values=None, filler_feature_ranges=None, ax=None, X_highlight=None, zoom_factor=1.0, legend=1, hide_spines=True, markers='s^oxv<>', colors='#1f77b4,#ff7f0e,#3ca02c,#d62728,#9467bd,#8c564b,#e377c2,#7f7f7f,#bcbd22,#17becf', scatter_kwargs=None, contourf_kwargs=None, contour_kwargs=None, scatter_highlight_kwargs=None, n_jobs=None) Plot decision regions of a classifier. Please note that this functions assumes that class labels are labeled consecutively, e.g,. 0, 1, 2, 3, 4, and 5. If you have class labels with integer labels > 4, you may want to provide additional colors and/or markers as `colors` and `markers` arguments. See https://matplotlib.org/examples/color/named_colors.html for more information. Parameters X : array-like, shape = [n_samples, n_features] Feature Matrix. y : array-like, shape = [n_samples] True class labels. clf : Classifier object. Must have a .predict method. feature_index : array-like (default: (0,) for 1D, (0, 1) otherwise) Feature indices to use for plotting. The first index in feature_index will be on the x-axis, the second index will be on the y-axis. filler_feature_values : dict (default: None) Only needed for number features > 2. Dictionary of feature index-value pairs for the features not being plotted. filler_feature_ranges : dict (default: None) Only needed for number features > 2. Dictionary of feature index-value pairs for the features not being plotted. Will use the ranges provided to select training samples for plotting. ax : matplotlib.axes.Axes (default: None) An existing matplotlib Axes. Creates one if ax=None. X_highlight : array-like, shape = [n_samples, n_features] (default: None) An array with data points that are used to highlight samples in X . zoom_factor : float (default: 1.0) Controls the scale of the x- and y-axis of the decision plot. hide_spines : bool (default: True) Hide axis spines if True. legend : int (default: 1) Integer to specify the legend location. No legend if legend is 0. markers : str (default: 's^oxv<>') Scatterplot markers. colors : str (default: 'red,blue,limegreen,gray,cyan') Comma separated list of colors. scatter_kwargs : dict (default: None) Keyword arguments for underlying matplotlib scatter function. contourf_kwargs : dict (default: None) Keyword arguments for underlying matplotlib contourf function. contour_kwargs : dict (default: None) Keyword arguments for underlying matplotlib contour function (which draws the lines between decision regions). scatter_highlight_kwargs : dict (default: None) Keyword arguments for underlying matplotlib scatter function. n_jobs : int or None, optional (default=None) The number of CPUs to use to do the computation using Python's multiprocessing library. None means 1. -1 means using all processors. New in v0.22.0. Returns ax : matplotlib.axes.Axes object Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/plotting/plot_decision_regions/","title":"plot_decision_regions: Visualize the decision regions of a classifier"},{"location":"user_guide/plotting/plot_decision_regions/#plot_decision_regions-visualize-the-decision-regions-of-a-classifier","text":"A function for plotting decision regions of classifiers in 1 or 2 dimensions. from mlxtend.plotting import plot_decision_regions","title":"plot_decision_regions: Visualize the decision regions of a classifier"},{"location":"user_guide/plotting/plot_decision_regions/#references","text":"","title":"References"},{"location":"user_guide/plotting/plot_decision_regions/#example-1-decision-regions-in-2d","text":"from mlxtend.plotting import plot_decision_regions import matplotlib.pyplot as plt from sklearn import datasets from sklearn.svm import SVC # Loading some example data iris = datasets.load_iris() X = iris.data[:, [0, 2]] y = iris.target # Training a classifier svm = SVC(C=0.5, kernel='linear') svm.fit(X, y) # Plotting decision regions plot_decision_regions(X, y, clf=svm, legend=2) # Adding axes annotations plt.xlabel('sepal length [cm]') plt.ylabel('petal length [cm]') plt.title('SVM on Iris') plt.show()","title":"Example 1 - Decision regions in 2D"},{"location":"user_guide/plotting/plot_decision_regions/#example-2-decision-regions-in-1d","text":"from mlxtend.plotting import plot_decision_regions import matplotlib.pyplot as plt from sklearn import datasets from sklearn.svm import SVC # Loading some example data iris = datasets.load_iris() X = iris.data[:, 2] X = X[:, None] y = iris.target # Training a classifier svm = SVC(C=0.5, kernel='linear') svm.fit(X, y) # Plotting decision regions plot_decision_regions(X, y, clf=svm, legend=2) # Adding axes annotations plt.xlabel('sepal length [cm]') plt.title('SVM on Iris') plt.show()","title":"Example 2 - Decision regions in 1D"},{"location":"user_guide/plotting/plot_decision_regions/#example-3-decision-region-grids","text":"from sklearn.linear_model import LogisticRegression from sklearn.naive_bayes import GaussianNB from sklearn.ensemble import RandomForestClassifier from sklearn.svm import SVC from sklearn import datasets import numpy as np # Initializing Classifiers clf1 = LogisticRegression(random_state=1, solver='newton-cg', multi_class='multinomial') clf2 = RandomForestClassifier(random_state=1, n_estimators=100) clf3 = GaussianNB() clf4 = SVC(gamma='auto') # Loading some example data iris = datasets.load_iris() X = iris.data[:, [0,2]] y = iris.target import matplotlib.pyplot as plt from mlxtend.plotting import plot_decision_regions import matplotlib.gridspec as gridspec import itertools gs = gridspec.GridSpec(2, 2) fig = plt.figure(figsize=(10,8)) labels = ['Logistic Regression', 'Random Forest', 'Naive Bayes', 'SVM'] for clf, lab, grd in zip([clf1, clf2, clf3, clf4], labels, itertools.product([0, 1], repeat=2)): clf.fit(X, y) ax = plt.subplot(gs[grd[0], grd[1]]) fig = plot_decision_regions(X=X, y=y, clf=clf, legend=2) plt.title(lab) plt.show()","title":"Example 3 - Decision Region Grids"},{"location":"user_guide/plotting/plot_decision_regions/#example-4-highlighting-test-data-points","text":"from mlxtend.plotting import plot_decision_regions from mlxtend.preprocessing import shuffle_arrays_unison import matplotlib.pyplot as plt from sklearn import datasets from sklearn.svm import SVC # Loading some example data iris = datasets.load_iris() X, y = iris.data[:, [0,2]], iris.target X, y = shuffle_arrays_unison(arrays=[X, y], random_seed=3) X_train, y_train = X[:100], y[:100] X_test, y_test = X[100:], y[100:] # Training a classifier svm = SVC(C=0.5, kernel='linear') svm.fit(X_train, y_train) # Plotting decision regions plot_decision_regions(X, y, clf=svm, legend=2, X_highlight=X_test) # Adding axes annotations plt.xlabel('sepal length [cm]') plt.ylabel('petal length [cm]') plt.title('SVM on Iris') plt.show()","title":"Example 4 - Highlighting Test Data Points"},{"location":"user_guide/plotting/plot_decision_regions/#example-5-evaluating-classifier-behavior-on-non-linear-problems","text":"from sklearn.linear_model import LogisticRegression from sklearn.naive_bayes import GaussianNB from sklearn.ensemble import RandomForestClassifier from sklearn.svm import SVC # Initializing Classifiers clf1 = LogisticRegression(random_state=1, solver='lbfgs') clf2 = RandomForestClassifier(n_estimators=100, random_state=1) clf3 = GaussianNB() clf4 = SVC(gamma='auto') # Loading Plotting Utilities import matplotlib.pyplot as plt import matplotlib.gridspec as gridspec import itertools from mlxtend.plotting import plot_decision_regions import numpy as np","title":"Example 5 - Evaluating Classifier Behavior on Non-Linear Problems"},{"location":"user_guide/plotting/plot_decision_regions/#xor","text":"xx, yy = np.meshgrid(np.linspace(-3, 3, 50), np.linspace(-3, 3, 50)) rng = np.random.RandomState(0) X = rng.randn(300, 2) y = np.array(np.logical_xor(X[:, 0] > 0, X[:, 1] > 0), dtype=int) gs = gridspec.GridSpec(2, 2) fig = plt.figure(figsize=(10,8)) labels = ['Logistic Regression', 'Random Forest', 'Naive Bayes', 'SVM'] for clf, lab, grd in zip([clf1, clf2, clf3, clf4], labels, itertools.product([0, 1], repeat=2)): clf.fit(X, y) ax = plt.subplot(gs[grd[0], grd[1]]) fig = plot_decision_regions(X=X, y=y, clf=clf, legend=2) plt.title(lab) plt.show()","title":"XOR"},{"location":"user_guide/plotting/plot_decision_regions/#half-moons","text":"from sklearn.datasets import make_moons X, y = make_moons(n_samples=100, random_state=123) gs = gridspec.GridSpec(2, 2) fig = plt.figure(figsize=(10,8)) labels = ['Logistic Regression', 'Random Forest', 'Naive Bayes', 'SVM'] for clf, lab, grd in zip([clf1, clf2, clf3, clf4], labels, itertools.product([0, 1], repeat=2)): clf.fit(X, y) ax = plt.subplot(gs[grd[0], grd[1]]) fig = plot_decision_regions(X=X, y=y, clf=clf, legend=2) plt.title(lab) plt.show()","title":"Half-Moons"},{"location":"user_guide/plotting/plot_decision_regions/#concentric-circles","text":"from sklearn.datasets import make_circles X, y = make_circles(n_samples=1000, random_state=123, noise=0.1, factor=0.2) gs = gridspec.GridSpec(2, 2) fig = plt.figure(figsize=(10,8)) labels = ['Logistic Regression', 'Random Forest', 'Naive Bayes', 'SVM'] for clf, lab, grd in zip([clf1, clf2, clf3, clf4], labels, itertools.product([0, 1], repeat=2)): clf.fit(X, y) ax = plt.subplot(gs[grd[0], grd[1]]) fig = plot_decision_regions(X=X, y=y, clf=clf, legend=2) plt.title(lab) plt.show()","title":"Concentric Circles"},{"location":"user_guide/plotting/plot_decision_regions/#example-6-working-with-existing-axes-objects-using-subplots","text":"import matplotlib.pyplot as plt from mlxtend.plotting import plot_decision_regions from sklearn.linear_model import LogisticRegression from sklearn.naive_bayes import GaussianNB from sklearn import datasets import numpy as np # Loading some example data iris = datasets.load_iris() X = iris.data[:, 2] X = X[:, None] y = iris.target # Initializing and fitting classifiers clf1 = LogisticRegression(random_state=1, solver='lbfgs', multi_class='multinomial') clf2 = GaussianNB() clf1.fit(X, y) clf2.fit(X, y) fig, axes = plt.subplots(1, 2, figsize=(10, 3)) fig = plot_decision_regions(X=X, y=y, clf=clf1, ax=axes[0], legend=2) fig = plot_decision_regions(X=X, y=y, clf=clf2, ax=axes[1], legend=1) plt.show()","title":"Example 6 - Working with existing axes objects using subplots"},{"location":"user_guide/plotting/plot_decision_regions/#example-7-decision-regions-with-more-than-two-training-features","text":"from mlxtend.plotting import plot_decision_regions import matplotlib.pyplot as plt from sklearn import datasets from sklearn.svm import SVC # Loading some example data X, y = datasets.make_blobs(n_samples=600, n_features=3, centers=[[2, 2, -2],[-2, -2, 2]], cluster_std=[2, 2], random_state=2) # Training a classifier svm = SVC(gamma='auto') svm.fit(X, y) # Plotting decision regions fig, ax = plt.subplots() # Decision region for feature 3 = 1.5 value = 1.5 # Plot training sample with feature 3 = 1.5 +/- 0.75 width = 0.75 plot_decision_regions(X, y, clf=svm, filler_feature_values={2: value}, filler_feature_ranges={2: width}, legend=2, ax=ax) ax.set_xlabel('Feature 1') ax.set_ylabel('Feature 2') ax.set_title('Feature 3 = {}'.format(value)) # Adding axes annotations fig.suptitle('SVM on make_blobs') plt.show()","title":"Example 7 - Decision regions with more than two training features"},{"location":"user_guide/plotting/plot_decision_regions/#example-8-grid-of-decision-region-slices","text":"from mlxtend.plotting import plot_decision_regions import matplotlib.pyplot as plt from sklearn import datasets from sklearn.svm import SVC # Loading some example data X, y = datasets.make_blobs(n_samples=500, n_features=3, centers=[[2, 2, -2],[-2, -2, 2]], cluster_std=[2, 2], random_state=2) # Training a classifier svm = SVC(gamma='auto') svm.fit(X, y) # Plotting decision regions fig, axarr = plt.subplots(2, 2, figsize=(10,8), sharex=True, sharey=True) values = [-4.0, -1.0, 1.0, 4.0] width = 0.75 for value, ax in zip(values, axarr.flat): plot_decision_regions(X, y, clf=svm, filler_feature_values={2: value}, filler_feature_ranges={2: width}, legend=2, ax=ax) ax.set_xlabel('Feature 1') ax.set_ylabel('Feature 2') ax.set_title('Feature 3 = {}'.format(value)) # Adding axes annotations fig.suptitle('SVM on make_blobs') plt.show()","title":"Example 8 - Grid of decision region slices"},{"location":"user_guide/plotting/plot_decision_regions/#example-9-customizing-the-plotting-style","text":"from mlxtend.plotting import plot_decision_regions from mlxtend.preprocessing import shuffle_arrays_unison import matplotlib.pyplot as plt from sklearn import datasets from sklearn.svm import SVC # Loading some example data iris = datasets.load_iris() X = iris.data[:, [0, 2]] y = iris.target X, y = shuffle_arrays_unison(arrays=[X, y], random_seed=3) X_train, y_train = X[:100], y[:100] X_test, y_test = X[100:], y[100:] # Training a classifier svm = SVC(C=0.5, kernel='linear') svm.fit(X_train, y_train) # Specify keyword arguments to be passed to underlying plotting functions scatter_kwargs = {'s': 120, 'edgecolor': None, 'alpha': 0.7} contourf_kwargs = {'alpha': 0.2} scatter_highlight_kwargs = {'s': 120, 'label': 'Test data', 'alpha': 0.7} # Plotting decision regions plot_decision_regions(X, y, clf=svm, legend=2, X_highlight=X_test, scatter_kwargs=scatter_kwargs, contourf_kwargs=contourf_kwargs, scatter_highlight_kwargs=scatter_highlight_kwargs) # Adding axes annotations plt.xlabel('sepal length [cm]') plt.ylabel('petal length [cm]') plt.title('SVM on Iris') plt.show()","title":"Example 9 - Customizing the plotting style"},{"location":"user_guide/plotting/plot_decision_regions/#example-10-providing-your-own-legend-labels","text":"Custom legend labels can be provided by returning the axis object(s) from the plot_decision_region function and then getting the handles and labels of the legend. Custom handles (i.e., labels) can then be provided via ax.legend ax = plot_decision_regions(X, y, clf=svm, legend=0) handles, labels = ax.get_legend_handles_labels() ax.legend(handles, ['class 0', 'class 1', 'class 2'], framealpha=0.3, scatterpoints=1) An example is shown below. from mlxtend.plotting import plot_decision_regions import matplotlib.pyplot as plt from sklearn import datasets from sklearn.svm import SVC # Loading some example data iris = datasets.load_iris() X = iris.data[:, [0, 2]] y = iris.target # Training a classifier svm = SVC(C=0.5, kernel='linear') svm.fit(X, y) # Plotting decision regions ax = plot_decision_regions(X, y, clf=svm, legend=0) # Adding axes annotations plt.xlabel('sepal length [cm]') plt.ylabel('petal length [cm]') plt.title('SVM on Iris') handles, labels = ax.get_legend_handles_labels() ax.legend(handles, ['class square', 'class triangle', 'class circle'], framealpha=0.3, scatterpoints=1) plt.show()","title":"Example 10 - Providing your own legend labels"},{"location":"user_guide/plotting/plot_decision_regions/#example-11-plots-with-different-zoom-factors","text":"from mlxtend.plotting import plot_decision_regions import matplotlib.pyplot as plt from sklearn import datasets from sklearn.svm import SVC # Loading some example data iris = datasets.load_iris() X = iris.data[:, [0, 2]] y = iris.target # Training a classifier svm = SVC(C=0.5, kernel='linear') svm.fit(X, y) #sk-container-id-1 {color: black;background-color: white;}#sk-container-id-1 pre{padding: 0;}#sk-container-id-1 div.sk-toggleable {background-color: white;}#sk-container-id-1 label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.3em;box-sizing: border-box;text-align: center;}#sk-container-id-1 label.sk-toggleable__label-arrow:before {content: \"\u25b8\";float: left;margin-right: 0.25em;color: #696969;}#sk-container-id-1 label.sk-toggleable__label-arrow:hover:before {color: black;}#sk-container-id-1 div.sk-estimator:hover label.sk-toggleable__label-arrow:before {color: black;}#sk-container-id-1 div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}#sk-container-id-1 div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}#sk-container-id-1 input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}#sk-container-id-1 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {content: \"\u25be\";}#sk-container-id-1 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}#sk-container-id-1 div.sk-estimator {font-family: monospace;background-color: #f0f8ff;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;margin-bottom: 0.5em;}#sk-container-id-1 div.sk-estimator:hover {background-color: #d4ebff;}#sk-container-id-1 div.sk-parallel-item::after {content: \"\";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}#sk-container-id-1 div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 div.sk-serial::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: 0;}#sk-container-id-1 div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;padding-right: 0.2em;padding-left: 0.2em;position: relative;}#sk-container-id-1 div.sk-item {position: relative;z-index: 1;}#sk-container-id-1 div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;position: relative;}#sk-container-id-1 div.sk-item::before, #sk-container-id-1 div.sk-parallel-item::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: -1;}#sk-container-id-1 div.sk-parallel-item {display: flex;flex-direction: column;z-index: 1;position: relative;background-color: white;}#sk-container-id-1 div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}#sk-container-id-1 div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}#sk-container-id-1 div.sk-parallel-item:only-child::after {width: 0;}#sk-container-id-1 div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0 0.4em 0.5em 0.4em;box-sizing: border-box;padding-bottom: 0.4em;background-color: white;}#sk-container-id-1 div.sk-label label {font-family: monospace;font-weight: bold;display: inline-block;line-height: 1.2em;}#sk-container-id-1 div.sk-label-container {text-align: center;}#sk-container-id-1 div.sk-container {/* jupyter's `normalize.less` sets `[hidden] { display: none; }` but bootstrap.min.css set `[hidden] { display: none !important; }` so we also need the `!important` here to be able to override the default hidden behavior on the sphinx rendered scikit-learn.org. See: https://github.com/scikit-learn/scikit-learn/issues/21755 */display: inline-block !important;position: relative;}#sk-container-id-1 div.sk-text-repr-fallback {display: none;} SVC(C=0.5, kernel='linear') In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org. SVC SVC(C=0.5, kernel='linear') Default Zoom Factor plot_decision_regions(X, y, clf=svm, zoom_factor=1.) plt.show() Zooming out plot_decision_regions(X, y, clf=svm, zoom_factor=0.1) plt.show() Zooming in Note that while zooming in (by choosing a zoom_factor > 1.0) the plots are still created such that all data points are shown in the plot. plot_decision_regions(X, y, clf=svm, zoom_factor=2.0) plt.show() Cropping the axes In order to zoom in further, which means that some training examples won't be shown, you can simply crop the axes as shown below: plot_decision_regions(X, y, clf=svm, zoom_factor=2.0) plt.xlim(5, 6) plt.ylim(2, 5) plt.show()","title":"Example 11 - Plots with different zoom factors"},{"location":"user_guide/plotting/plot_decision_regions/#example-12-using-classifiers-that-expect-onehot-encoded-outputs-keras","text":"Most objects for classification that mimick the scikit-learn estimator API should be compatible with the plot_decision_regions function. However, if the classification model (e.g., a typical Keras model) output onehot-encoded predictions, we have to use an additional trick. I.e., for onehot encoded outputs, we need to wrap the Keras model into a class that converts these onehot encoded variables into integers. Such a wrapper class can be as simple as the following: class Onehot2Int(object): def __init__(self, model): self.model = model def predict(self, X): y_pred = self.model.predict(X) return np.argmax(y_pred, axis=1) The example below illustrates how the Onehot2Int class can be used with a Keras model that outputs onehot encoded labels: import keras from keras.models import Sequential from keras.layers import Dense import matplotlib.pyplot as plt import numpy as np from mlxtend.data import iris_data from mlxtend.preprocessing import standardize from mlxtend.plotting import plot_decision_regions from keras.utils import to_categorical X, y = iris_data() X = X[:, [2, 3]] X = standardize(X) # OneHot encoding y_onehot = to_categorical(y) # Create the model np.random.seed(123) model = Sequential() model.add(Dense(8, input_shape=(2,), activation='relu', kernel_initializer='he_uniform')) model.add(Dense(4, activation='relu', kernel_initializer='he_uniform')) model.add(Dense(3, activation='softmax')) # Configure the model and start training model.compile(loss=\"categorical_crossentropy\", optimizer=keras.optimizers.Adam(lr=0.005), metrics=['accuracy']) history = model.fit(X, y_onehot, epochs=10, batch_size=5, verbose=1, validation_split=0.1) Epoch 1/10 1/27 [>.............................] - ETA: 3s - loss: 1.2769 - accuracy: 0.4000 /Users/sebastianraschka/miniforge3/envs/mlxtend/lib/python3.8/site-packages/keras/optimizers/optimizer_v2/adam.py:117: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead. super().__init__(name, **kwargs) 2023-03-28 17:48:13.901264: W tensorflow/tsl/platform/profile_utils/cpu_utils.cc:128] Failed to get CPU frequency: 0 Hz 27/27 [==============================] - 0s 3ms/step - loss: 0.9526 - accuracy: 0.4222 - val_loss: 1.2656 - val_accuracy: 0.0000e+00 Epoch 2/10 27/27 [==============================] - 0s 834us/step - loss: 0.7062 - accuracy: 0.6741 - val_loss: 1.0939 - val_accuracy: 0.0000e+00 Epoch 3/10 27/27 [==============================] - 0s 808us/step - loss: 0.6461 - accuracy: 0.7111 - val_loss: 1.0705 - val_accuracy: 0.0667 Epoch 4/10 27/27 [==============================] - 0s 767us/step - loss: 0.6145 - accuracy: 0.7185 - val_loss: 1.0518 - val_accuracy: 0.0000e+00 Epoch 5/10 27/27 [==============================] - 0s 746us/step - loss: 0.5877 - accuracy: 0.7185 - val_loss: 1.0470 - val_accuracy: 0.0000e+00 Epoch 6/10 27/27 [==============================] - 0s 740us/step - loss: 0.5496 - accuracy: 0.7333 - val_loss: 1.0275 - val_accuracy: 0.0000e+00 Epoch 7/10 27/27 [==============================] - 0s 734us/step - loss: 0.4985 - accuracy: 0.7333 - val_loss: 1.0131 - val_accuracy: 0.0000e+00 Epoch 8/10 27/27 [==============================] - 0s 739us/step - loss: 0.4365 - accuracy: 0.7333 - val_loss: 0.9634 - val_accuracy: 0.0000e+00 Epoch 9/10 27/27 [==============================] - 0s 729us/step - loss: 0.3875 - accuracy: 0.7333 - val_loss: 0.9442 - val_accuracy: 0.0000e+00 Epoch 10/10 27/27 [==============================] - 0s 764us/step - loss: 0.3402 - accuracy: 0.7407 - val_loss: 0.8565 - val_accuracy: 0.0000e+00 # Wrap keras model model_no_ohe = Onehot2Int(model) # Plot decision boundary plot_decision_regions(X, y, clf=model_no_ohe) plt.show() 9600/9600 [==============================] - 3s 289us/step","title":"Example 12 - Using classifiers that expect onehot-encoded outputs (Keras)"},{"location":"user_guide/plotting/plot_decision_regions/#api","text":"plot_decision_regions(X, y, clf, feature_index=None, filler_feature_values=None, filler_feature_ranges=None, ax=None, X_highlight=None, zoom_factor=1.0, legend=1, hide_spines=True, markers='s^oxv<>', colors='#1f77b4,#ff7f0e,#3ca02c,#d62728,#9467bd,#8c564b,#e377c2,#7f7f7f,#bcbd22,#17becf', scatter_kwargs=None, contourf_kwargs=None, contour_kwargs=None, scatter_highlight_kwargs=None, n_jobs=None) Plot decision regions of a classifier. Please note that this functions assumes that class labels are labeled consecutively, e.g,. 0, 1, 2, 3, 4, and 5. If you have class labels with integer labels > 4, you may want to provide additional colors and/or markers as `colors` and `markers` arguments. See https://matplotlib.org/examples/color/named_colors.html for more information. Parameters X : array-like, shape = [n_samples, n_features] Feature Matrix. y : array-like, shape = [n_samples] True class labels. clf : Classifier object. Must have a .predict method. feature_index : array-like (default: (0,) for 1D, (0, 1) otherwise) Feature indices to use for plotting. The first index in feature_index will be on the x-axis, the second index will be on the y-axis. filler_feature_values : dict (default: None) Only needed for number features > 2. Dictionary of feature index-value pairs for the features not being plotted. filler_feature_ranges : dict (default: None) Only needed for number features > 2. Dictionary of feature index-value pairs for the features not being plotted. Will use the ranges provided to select training samples for plotting. ax : matplotlib.axes.Axes (default: None) An existing matplotlib Axes. Creates one if ax=None. X_highlight : array-like, shape = [n_samples, n_features] (default: None) An array with data points that are used to highlight samples in X . zoom_factor : float (default: 1.0) Controls the scale of the x- and y-axis of the decision plot. hide_spines : bool (default: True) Hide axis spines if True. legend : int (default: 1) Integer to specify the legend location. No legend if legend is 0. markers : str (default: 's^oxv<>') Scatterplot markers. colors : str (default: 'red,blue,limegreen,gray,cyan') Comma separated list of colors. scatter_kwargs : dict (default: None) Keyword arguments for underlying matplotlib scatter function. contourf_kwargs : dict (default: None) Keyword arguments for underlying matplotlib contourf function. contour_kwargs : dict (default: None) Keyword arguments for underlying matplotlib contour function (which draws the lines between decision regions). scatter_highlight_kwargs : dict (default: None) Keyword arguments for underlying matplotlib scatter function. n_jobs : int or None, optional (default=None) The number of CPUs to use to do the computation using Python's multiprocessing library. None means 1. -1 means using all processors. New in v0.22.0. Returns ax : matplotlib.axes.Axes object Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/plotting/plot_decision_regions/","title":"API"},{"location":"user_guide/plotting/plot_learning_curves/","text":"plot_learning_curves: Plot learning curves from training and test sets A function to plot learning curves for classifiers. Learning curves are extremely useful to analyze if a model is suffering from over- or under-fitting (high variance or high bias). The function can be imported via from mlxtend.plotting import plot_learning_curves This function uses the traditional holdout method based on a training and a test (or validation) set. The test set is kept constant while the size of the training set is increased gradually. The model is fit on the training set (of varying size) and evaluated on the same test set. The learning curve can be used as follows to diagnose overfitting: If there is a large gap between the training and test performance, then the model is likely suffering from overfitting. If both the training and test error are very large, the model is likely underfitting the data. The learning curve can also be used to judge whether collecting more data can be useful. More about that in Example 1 below. References - Example 1 The following code illustrates how we can construct a learning curve for a 5000-sample subset of the MNIST dataset. 4000 examples are used for training, and 1000 examples are reserved for testing. from mlxtend.plotting import plot_learning_curves import matplotlib.pyplot as plt from mlxtend.data import mnist_data from mlxtend.preprocessing import shuffle_arrays_unison from sklearn.neighbors import KNeighborsClassifier import numpy as np # Loading some example data X, y = mnist_data() X, y = shuffle_arrays_unison(arrays=[X, y], random_seed=123) X_train, X_test = X[:4000], X[4000:] y_train, y_test = y[:4000], y[4000:] clf = KNeighborsClassifier(n_neighbors=7) plot_learning_curves(X_train, y_train, X_test, y_test, clf) plt.show() As we can see from the plot above, the KNN model could benefit from additional training data. I.e., the slope indicates that if we had a larger training set, the test set error might decrease further. Also, based on the gap between training and test set performance, the model overfits slightly. This could pontentially be addressed by increasing the number of neighbors ( n_neighbors ) in KNN. While it is not relevant for analyzing the performance of the classifier, the region ~20% training set size shows that the model is underfitting (large training and test error), which is probably due to the small dataset size. API plot_learning_curves(X_train, y_train, X_test, y_test, clf, train_marker='o', test_marker='^', scoring='misclassification error', suppress_plot=False, print_model=True, title_fontsize=12, style='default', legend_loc='best') Plots learning curves of a classifier. Parameters X_train : array-like, shape = [n_samples, n_features] Feature matrix of the training dataset. y_train : array-like, shape = [n_samples] True class labels of the training dataset. X_test : array-like, shape = [n_samples, n_features] Feature matrix of the test dataset. y_test : array-like, shape = [n_samples] True class labels of the test dataset. clf : Classifier object. Must have a .predict .fit method. train_marker : str (default: 'o') Marker for the training set line plot. test_marker : str (default: '^') Marker for the test set line plot. scoring : str (default: 'misclassification error') If not 'misclassification error', accepts the following metrics (from scikit-learn): {'accuracy', 'average_precision', 'f1_micro', 'f1_macro', 'f1_weighted', 'f1_samples', 'log_loss', 'precision', 'recall', 'roc_auc', 'adjusted_rand_score', 'mean_absolute_error', 'mean_squared_error', 'median_absolute_error', 'r2'} suppress_plot=False : bool (default: False) Suppress matplotlib plots if True. Recommended for testing purposes. print_model : bool (default: True) Print model parameters in plot title if True. style : str (default: 'default') Matplotlib style. For more styles, please see https://matplotlib.org/stable/gallery/style_sheets/style_sheets_reference.html legend_loc : str (default: 'best') Where to place the plot legend: {'best', 'upper left', 'upper right', 'lower left', 'lower right'} Returns errors : (training_error, test_error): tuple of lists Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/plotting/plot_learning_curves/","title":"plot_learning_curves: Plot learning curves from training and test sets"},{"location":"user_guide/plotting/plot_learning_curves/#plot_learning_curves-plot-learning-curves-from-training-and-test-sets","text":"A function to plot learning curves for classifiers. Learning curves are extremely useful to analyze if a model is suffering from over- or under-fitting (high variance or high bias). The function can be imported via from mlxtend.plotting import plot_learning_curves This function uses the traditional holdout method based on a training and a test (or validation) set. The test set is kept constant while the size of the training set is increased gradually. The model is fit on the training set (of varying size) and evaluated on the same test set. The learning curve can be used as follows to diagnose overfitting: If there is a large gap between the training and test performance, then the model is likely suffering from overfitting. If both the training and test error are very large, the model is likely underfitting the data. The learning curve can also be used to judge whether collecting more data can be useful. More about that in Example 1 below.","title":"plot_learning_curves: Plot learning curves from training and test sets"},{"location":"user_guide/plotting/plot_learning_curves/#references","text":"-","title":"References"},{"location":"user_guide/plotting/plot_learning_curves/#example-1","text":"The following code illustrates how we can construct a learning curve for a 5000-sample subset of the MNIST dataset. 4000 examples are used for training, and 1000 examples are reserved for testing. from mlxtend.plotting import plot_learning_curves import matplotlib.pyplot as plt from mlxtend.data import mnist_data from mlxtend.preprocessing import shuffle_arrays_unison from sklearn.neighbors import KNeighborsClassifier import numpy as np # Loading some example data X, y = mnist_data() X, y = shuffle_arrays_unison(arrays=[X, y], random_seed=123) X_train, X_test = X[:4000], X[4000:] y_train, y_test = y[:4000], y[4000:] clf = KNeighborsClassifier(n_neighbors=7) plot_learning_curves(X_train, y_train, X_test, y_test, clf) plt.show() As we can see from the plot above, the KNN model could benefit from additional training data. I.e., the slope indicates that if we had a larger training set, the test set error might decrease further. Also, based on the gap between training and test set performance, the model overfits slightly. This could pontentially be addressed by increasing the number of neighbors ( n_neighbors ) in KNN. While it is not relevant for analyzing the performance of the classifier, the region ~20% training set size shows that the model is underfitting (large training and test error), which is probably due to the small dataset size.","title":"Example 1"},{"location":"user_guide/plotting/plot_learning_curves/#api","text":"plot_learning_curves(X_train, y_train, X_test, y_test, clf, train_marker='o', test_marker='^', scoring='misclassification error', suppress_plot=False, print_model=True, title_fontsize=12, style='default', legend_loc='best') Plots learning curves of a classifier. Parameters X_train : array-like, shape = [n_samples, n_features] Feature matrix of the training dataset. y_train : array-like, shape = [n_samples] True class labels of the training dataset. X_test : array-like, shape = [n_samples, n_features] Feature matrix of the test dataset. y_test : array-like, shape = [n_samples] True class labels of the test dataset. clf : Classifier object. Must have a .predict .fit method. train_marker : str (default: 'o') Marker for the training set line plot. test_marker : str (default: '^') Marker for the test set line plot. scoring : str (default: 'misclassification error') If not 'misclassification error', accepts the following metrics (from scikit-learn): {'accuracy', 'average_precision', 'f1_micro', 'f1_macro', 'f1_weighted', 'f1_samples', 'log_loss', 'precision', 'recall', 'roc_auc', 'adjusted_rand_score', 'mean_absolute_error', 'mean_squared_error', 'median_absolute_error', 'r2'} suppress_plot=False : bool (default: False) Suppress matplotlib plots if True. Recommended for testing purposes. print_model : bool (default: True) Print model parameters in plot title if True. style : str (default: 'default') Matplotlib style. For more styles, please see https://matplotlib.org/stable/gallery/style_sheets/style_sheets_reference.html legend_loc : str (default: 'best') Where to place the plot legend: {'best', 'upper left', 'upper right', 'lower left', 'lower right'} Returns errors : (training_error, test_error): tuple of lists Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/plotting/plot_learning_curves/","title":"API"},{"location":"user_guide/plotting/plot_linear_regression/","text":"plot_linear_regression: A quick way for plotting linear regression fits A function to plot linear regression fits. from mlxtend.plotting import plot_linear_regression Overview The plot_linear_regression is a convenience function that uses scikit-learn's linear_model.LinearRegression to fit a linear model and SciPy's stats.pearsonr to calculate the correlation coefficient. References - Example 1 - Ordinary Least Squares Simple Linear Regression import matplotlib.pyplot as plt from mlxtend.plotting import plot_linear_regression import numpy as np X = np.array([4, 8, 13, 26, 31, 10, 8, 30, 18, 12, 20, 5, 28, 18, 6, 31, 12, 12, 27, 11, 6, 14, 25, 7, 13,4, 15, 21, 15]) y = np.array([14, 24, 22, 59, 66, 25, 18, 60, 39, 32, 53, 18, 55, 41, 28, 61, 35, 36, 52, 23, 19, 25, 73, 16, 32, 14, 31, 43, 34]) intercept, slope, corr_coeff = plot_linear_regression(X, y) plt.show() API plot_linear_regression(X, y, model=LinearRegression(copy_X=True, fit_intercept=True, n_jobs=1, normalize=False), corr_func='pearsonr', scattercolor='blue', fit_style='k--', legend=True, xlim='auto') Plot a linear regression line fit. Parameters X : numpy array, shape = [n_samples,] Samples. y : numpy array, shape (n_samples,) Target values model: object (default: sklearn.linear_model.LinearRegression) Estimator object for regression. Must implement a .fit() and .predict() method. corr_func: str or function (default: 'pearsonr') Uses pearsonr from scipy.stats if corr_func='pearsonr'. to compute the regression slope. If not 'pearsonr', the corr_func , the corr_func parameter expects a function of the form func( , ) as inputs, which is expected to return a tuple (<correlation_coefficient>, <some_unused_value>) . scattercolor: string (default: blue) Color of scatter plot points. fit_style: string (default: k--) Style for the line fit. legend: bool (default: True) Plots legend with corr_coeff coef., fit coef., and intercept values. xlim: array-like (x_min, x_max) or 'auto' (default: 'auto') X-axis limits for the linear line fit. Returns regression_fit : tuple intercept, slope, corr_coeff (float, float, float) Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/plotting/plot_linear_regression/","title":"plot_linear_regression: A quick way for plotting linear regression fits"},{"location":"user_guide/plotting/plot_linear_regression/#plot_linear_regression-a-quick-way-for-plotting-linear-regression-fits","text":"A function to plot linear regression fits. from mlxtend.plotting import plot_linear_regression","title":"plot_linear_regression: A quick way for plotting linear regression fits"},{"location":"user_guide/plotting/plot_linear_regression/#overview","text":"The plot_linear_regression is a convenience function that uses scikit-learn's linear_model.LinearRegression to fit a linear model and SciPy's stats.pearsonr to calculate the correlation coefficient.","title":"Overview"},{"location":"user_guide/plotting/plot_linear_regression/#references","text":"-","title":"References"},{"location":"user_guide/plotting/plot_linear_regression/#example-1-ordinary-least-squares-simple-linear-regression","text":"import matplotlib.pyplot as plt from mlxtend.plotting import plot_linear_regression import numpy as np X = np.array([4, 8, 13, 26, 31, 10, 8, 30, 18, 12, 20, 5, 28, 18, 6, 31, 12, 12, 27, 11, 6, 14, 25, 7, 13,4, 15, 21, 15]) y = np.array([14, 24, 22, 59, 66, 25, 18, 60, 39, 32, 53, 18, 55, 41, 28, 61, 35, 36, 52, 23, 19, 25, 73, 16, 32, 14, 31, 43, 34]) intercept, slope, corr_coeff = plot_linear_regression(X, y) plt.show()","title":"Example 1 - Ordinary Least Squares Simple Linear Regression"},{"location":"user_guide/plotting/plot_linear_regression/#api","text":"plot_linear_regression(X, y, model=LinearRegression(copy_X=True, fit_intercept=True, n_jobs=1, normalize=False), corr_func='pearsonr', scattercolor='blue', fit_style='k--', legend=True, xlim='auto') Plot a linear regression line fit. Parameters X : numpy array, shape = [n_samples,] Samples. y : numpy array, shape (n_samples,) Target values model: object (default: sklearn.linear_model.LinearRegression) Estimator object for regression. Must implement a .fit() and .predict() method. corr_func: str or function (default: 'pearsonr') Uses pearsonr from scipy.stats if corr_func='pearsonr'. to compute the regression slope. If not 'pearsonr', the corr_func , the corr_func parameter expects a function of the form func( , ) as inputs, which is expected to return a tuple (<correlation_coefficient>, <some_unused_value>) . scattercolor: string (default: blue) Color of scatter plot points. fit_style: string (default: k--) Style for the line fit. legend: bool (default: True) Plots legend with corr_coeff coef., fit coef., and intercept values. xlim: array-like (x_min, x_max) or 'auto' (default: 'auto') X-axis limits for the linear line fit. Returns regression_fit : tuple intercept, slope, corr_coeff (float, float, float) Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/plotting/plot_linear_regression/","title":"API"},{"location":"user_guide/plotting/plot_pca_correlation_graph/","text":"plot_pca_correlation_graph: plot correlations between original features and principal components A function to provide a correlation circle for PCA. > from mlxtend.plotting import plot_pca_correlation_graph In a so called correlation circle, the correlations between the original dataset features and the principal component(s) are shown via coordinates. Example The following correlation circle examples visualizes the correlation between the first two principal components and the 4 original iris dataset features. Features with a positive correlation will be grouped together. Totally uncorrelated features are orthogonal to each other. Features with a negative correlation will be plotted on the opposing quadrants of this plot. from mlxtend.data import iris_data from mlxtend.plotting import plot_pca_correlation_graph import numpy as np X, y = iris_data() X_norm = X / X.std(axis=0) # Normalizing the feature columns is recommended feature_names = [ 'sepal length', 'sepal width', 'petal length', 'petal width'] figure, correlation_matrix = plot_pca_correlation_graph(X_norm, feature_names, dimensions=(1, 2), figure_axis_size=10) correlation_matrix .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } Dim 1 Dim 2 sepal length -0.891224 -0.357352 sepal width 0.449313 -0.888351 petal length -0.991684 -0.020247 petal width -0.964996 -0.062786 Further, note that the percentage values shown on the x and y axis denote how much of the variance in the original dataset is explained by each principal component axis. I.e.., if PC1 lists 72.7% and PC2 lists 23.0% as shown above, then combined, the 2 principal components explain 95.7% of the total variance. API plot_pca_correlation_graph(X, variables_names, dimensions=(1, 2), figure_axis_size=6, X_pca=None, explained_variance=None) Compute the PCA for X and plots the Correlation graph Parameters X : 2d array like. The columns represent the different variables and the rows are the samples of thos variables variables_names : array like Name of the columns (the variables) of X dimensions: tuple with two elements. dimensions to be plotted (x,y) figure_axis_size : size of the final frame. The figure created is a square with length and width equal to figure_axis_size. X_pca : np.ndarray, shape = [n_samples, n_components]. Optional. X_pca is the matrix of the transformed components from X. If not provided, the function computes PCA automatically using mlxtend.feature_extraction.PrincipalComponentAnalysis Expected n_componentes >= max(dimensions) explained_variance : 1 dimension np.ndarray, length = n_components Optional. explained_variance are the eigenvalues from the diagonalized covariance matrix on the PCA transformatiopn. If not provided, the function computes PCA independently Expected n_componentes == X.shape[1] Returns matplotlib_figure, correlation_matrix Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/plotting/plot_pca_correlation_graph/","title":"plot_pca_correlation_graph: plot correlations between original features and principal components"},{"location":"user_guide/plotting/plot_pca_correlation_graph/#plot_pca_correlation_graph-plot-correlations-between-original-features-and-principal-components","text":"A function to provide a correlation circle for PCA. > from mlxtend.plotting import plot_pca_correlation_graph In a so called correlation circle, the correlations between the original dataset features and the principal component(s) are shown via coordinates.","title":"plot_pca_correlation_graph: plot correlations between original features and principal components"},{"location":"user_guide/plotting/plot_pca_correlation_graph/#example","text":"The following correlation circle examples visualizes the correlation between the first two principal components and the 4 original iris dataset features. Features with a positive correlation will be grouped together. Totally uncorrelated features are orthogonal to each other. Features with a negative correlation will be plotted on the opposing quadrants of this plot. from mlxtend.data import iris_data from mlxtend.plotting import plot_pca_correlation_graph import numpy as np X, y = iris_data() X_norm = X / X.std(axis=0) # Normalizing the feature columns is recommended feature_names = [ 'sepal length', 'sepal width', 'petal length', 'petal width'] figure, correlation_matrix = plot_pca_correlation_graph(X_norm, feature_names, dimensions=(1, 2), figure_axis_size=10) correlation_matrix .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } Dim 1 Dim 2 sepal length -0.891224 -0.357352 sepal width 0.449313 -0.888351 petal length -0.991684 -0.020247 petal width -0.964996 -0.062786 Further, note that the percentage values shown on the x and y axis denote how much of the variance in the original dataset is explained by each principal component axis. I.e.., if PC1 lists 72.7% and PC2 lists 23.0% as shown above, then combined, the 2 principal components explain 95.7% of the total variance.","title":"Example"},{"location":"user_guide/plotting/plot_pca_correlation_graph/#api","text":"plot_pca_correlation_graph(X, variables_names, dimensions=(1, 2), figure_axis_size=6, X_pca=None, explained_variance=None) Compute the PCA for X and plots the Correlation graph Parameters X : 2d array like. The columns represent the different variables and the rows are the samples of thos variables variables_names : array like Name of the columns (the variables) of X dimensions: tuple with two elements. dimensions to be plotted (x,y) figure_axis_size : size of the final frame. The figure created is a square with length and width equal to figure_axis_size. X_pca : np.ndarray, shape = [n_samples, n_components]. Optional. X_pca is the matrix of the transformed components from X. If not provided, the function computes PCA automatically using mlxtend.feature_extraction.PrincipalComponentAnalysis Expected n_componentes >= max(dimensions) explained_variance : 1 dimension np.ndarray, length = n_components Optional. explained_variance are the eigenvalues from the diagonalized covariance matrix on the PCA transformatiopn. If not provided, the function computes PCA independently Expected n_componentes == X.shape[1] Returns matplotlib_figure, correlation_matrix Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/plotting/plot_pca_correlation_graph/","title":"API"},{"location":"user_guide/plotting/plot_sequential_feature_selection/","text":"plot_sequential_feature_selection: Visualize selected feature subset performances from the SequentialFeatureSelector A matplotlib utility function for visualizing results from feature_selection.SequentialFeatureSelector . from mlxtend.plotting import plot_sequential_feature_selection Overview for more information on sequential feature selection, please see feature_selection.SequentialFeatureSelector . Example 1 - Plotting the results from SequentialFeatureSelector from mlxtend.plotting import plot_sequential_feature_selection as plot_sfs from mlxtend.feature_selection import SequentialFeatureSelector as SFS import matplotlib.pyplot as plt from sklearn.neighbors import KNeighborsClassifier from sklearn.datasets import load_iris iris = load_iris() X = iris.data y = iris.target knn = KNeighborsClassifier(n_neighbors=4) sfs = SFS(knn, k_features=4, forward=True, floating=False, scoring='accuracy', cv=5) sfs = sfs.fit(X, y) fig1 = plot_sfs(sfs.get_metric_dict(), kind='std_dev', figsize=(6, 4)) plt.ylim([0.8, 1]) plt.title('Sequential Forward Selection (w. StdDev)') plt.grid() plt.show() API plot_sequential_feature_selection(metric_dict, figsize=None, kind='std_dev', color='blue', bcolor='steelblue', marker='o', alpha=0.2, ylabel='Performance', confidence_interval=0.95) Plot feature selection results. Parameters metric_dict : mlxtend.SequentialFeatureSelector.get_metric_dict() object figsize : tuple (default: None) Height and width of the figure kind : str (default: \"std_dev\") The kind of error bar or confidence interval in {'std_dev', 'std_err', 'ci', None}. color : str (default: \"blue\") Color of the lineplot (accepts any matplotlib color name) bcolor : str (default: \"steelblue\"). Color of the error bars / confidence intervals (accepts any matplotlib color name). marker : str (default: \"o\") Marker of the line plot (accepts any matplotlib marker name). alpha : float in [0, 1] (default: 0.2) Transparency of the error bars / confidence intervals. ylabel : str (default: \"Performance\") Y-axis label. confidence_interval : float (default: 0.95) Confidence level if kind='ci' . Returns fig : matplotlib.pyplot.figure() object Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/plotting/plot_sequential_feature_selection/ ython","title":"plot_sequential_feature_selection: Visualize selected feature subset performances from the SequentialFeatureSelector"},{"location":"user_guide/plotting/plot_sequential_feature_selection/#plot_sequential_feature_selection-visualize-selected-feature-subset-performances-from-the-sequentialfeatureselector","text":"A matplotlib utility function for visualizing results from feature_selection.SequentialFeatureSelector . from mlxtend.plotting import plot_sequential_feature_selection","title":"plot_sequential_feature_selection: Visualize selected feature subset performances from the SequentialFeatureSelector"},{"location":"user_guide/plotting/plot_sequential_feature_selection/#overview","text":"for more information on sequential feature selection, please see feature_selection.SequentialFeatureSelector .","title":"Overview"},{"location":"user_guide/plotting/plot_sequential_feature_selection/#example-1-plotting-the-results-from-sequentialfeatureselector","text":"from mlxtend.plotting import plot_sequential_feature_selection as plot_sfs from mlxtend.feature_selection import SequentialFeatureSelector as SFS import matplotlib.pyplot as plt from sklearn.neighbors import KNeighborsClassifier from sklearn.datasets import load_iris iris = load_iris() X = iris.data y = iris.target knn = KNeighborsClassifier(n_neighbors=4) sfs = SFS(knn, k_features=4, forward=True, floating=False, scoring='accuracy', cv=5) sfs = sfs.fit(X, y) fig1 = plot_sfs(sfs.get_metric_dict(), kind='std_dev', figsize=(6, 4)) plt.ylim([0.8, 1]) plt.title('Sequential Forward Selection (w. StdDev)') plt.grid() plt.show()","title":"Example 1 - Plotting the results from SequentialFeatureSelector"},{"location":"user_guide/plotting/plot_sequential_feature_selection/#api","text":"plot_sequential_feature_selection(metric_dict, figsize=None, kind='std_dev', color='blue', bcolor='steelblue', marker='o', alpha=0.2, ylabel='Performance', confidence_interval=0.95) Plot feature selection results. Parameters metric_dict : mlxtend.SequentialFeatureSelector.get_metric_dict() object figsize : tuple (default: None) Height and width of the figure kind : str (default: \"std_dev\") The kind of error bar or confidence interval in {'std_dev', 'std_err', 'ci', None}. color : str (default: \"blue\") Color of the lineplot (accepts any matplotlib color name) bcolor : str (default: \"steelblue\"). Color of the error bars / confidence intervals (accepts any matplotlib color name). marker : str (default: \"o\") Marker of the line plot (accepts any matplotlib marker name). alpha : float in [0, 1] (default: 0.2) Transparency of the error bars / confidence intervals. ylabel : str (default: \"Performance\") Y-axis label. confidence_interval : float (default: 0.95) Confidence level if kind='ci' . Returns fig : matplotlib.pyplot.figure() object Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/plotting/plot_sequential_feature_selection/ ython","title":"API"},{"location":"user_guide/plotting/scatter_hist/","text":"scatter_hist: create a scatter histogram plot A function to quickly produce a scatter histogram plot. from mlxtend.plotting import scatter_hist Overview References https://matplotlib.org/gallery/lines_bars_and_markers/scatter_hist.html Example 1 - Scatter Plot and Histograms from Pandas DataFrames from mlxtend.data import iris_data from mlxtend.plotting import scatter_hist import pandas as pd X, y = iris_data() df = pd.DataFrame(X) df.columns = ['sepal length [cm]', 'sepal width [cm]', 'petal length [cm]', 'petal width [cm]'] df.head(5) .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } sepal length [cm] sepal width [cm] petal length [cm] petal width [cm] 0 5.1 3.5 1.4 0.2 1 4.9 3.0 1.4 0.2 2 4.7 3.2 1.3 0.2 3 4.6 3.1 1.5 0.2 4 5.0 3.6 1.4 0.2 import matplotlib.pyplot as plt from mlxtend.plotting import scatter_hist fig = scatter_hist(df[\"sepal length [cm]\"], df[\"sepal width [cm]\"]) Example 2 - Category Scatter from NumPy Arrays from mlxtend.data import iris_data from mlxtend.plotting import scatter_hist import pandas as pd X, y = iris_data() X[:5] array([[5.1, 3.5, 1.4, 0.2], [4.9, 3. , 1.4, 0.2], [4.7, 3.2, 1.3, 0.2], [4.6, 3.1, 1.5, 0.2], [5. , 3.6, 1.4, 0.2]]) fig = scatter_hist(X[:, 0], X[:, 1]) API scatter_hist(x, y, xlabel=None, ylabel=None, figsize=(5, 5)) Scatter plot and individual feature histograms along axes. Parameters x : 1D array-like or Pandas Series X-axis values. y : 1D array-like or Pandas Series Y-axis values. xlabel : str (default: None ) Label for the X-axis values. If x is a pandas Series, and xlabel is None , the label is inferred automatically. ylabel : str (default: None ) Label for the X-axis values. If y is a pandas Series, and ylabel is None , the label is inferred automatically. figsize : tuple (default: (5, 5) ) Matplotlib figure size. Returns plot : Matplotlib Figure object","title":"scatter_hist: create a scatter histogram plot"},{"location":"user_guide/plotting/scatter_hist/#scatter_hist-create-a-scatter-histogram-plot","text":"A function to quickly produce a scatter histogram plot. from mlxtend.plotting import scatter_hist","title":"scatter_hist: create a scatter histogram plot"},{"location":"user_guide/plotting/scatter_hist/#overview","text":"","title":"Overview"},{"location":"user_guide/plotting/scatter_hist/#references","text":"https://matplotlib.org/gallery/lines_bars_and_markers/scatter_hist.html","title":"References"},{"location":"user_guide/plotting/scatter_hist/#example-1-scatter-plot-and-histograms-from-pandas-dataframes","text":"from mlxtend.data import iris_data from mlxtend.plotting import scatter_hist import pandas as pd X, y = iris_data() df = pd.DataFrame(X) df.columns = ['sepal length [cm]', 'sepal width [cm]', 'petal length [cm]', 'petal width [cm]'] df.head(5) .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } sepal length [cm] sepal width [cm] petal length [cm] petal width [cm] 0 5.1 3.5 1.4 0.2 1 4.9 3.0 1.4 0.2 2 4.7 3.2 1.3 0.2 3 4.6 3.1 1.5 0.2 4 5.0 3.6 1.4 0.2 import matplotlib.pyplot as plt from mlxtend.plotting import scatter_hist fig = scatter_hist(df[\"sepal length [cm]\"], df[\"sepal width [cm]\"])","title":"Example 1 - Scatter Plot and Histograms from Pandas DataFrames"},{"location":"user_guide/plotting/scatter_hist/#example-2-category-scatter-from-numpy-arrays","text":"from mlxtend.data import iris_data from mlxtend.plotting import scatter_hist import pandas as pd X, y = iris_data() X[:5] array([[5.1, 3.5, 1.4, 0.2], [4.9, 3. , 1.4, 0.2], [4.7, 3.2, 1.3, 0.2], [4.6, 3.1, 1.5, 0.2], [5. , 3.6, 1.4, 0.2]]) fig = scatter_hist(X[:, 0], X[:, 1])","title":"Example 2 - Category Scatter from NumPy Arrays"},{"location":"user_guide/plotting/scatter_hist/#api","text":"scatter_hist(x, y, xlabel=None, ylabel=None, figsize=(5, 5)) Scatter plot and individual feature histograms along axes. Parameters x : 1D array-like or Pandas Series X-axis values. y : 1D array-like or Pandas Series Y-axis values. xlabel : str (default: None ) Label for the X-axis values. If x is a pandas Series, and xlabel is None , the label is inferred automatically. ylabel : str (default: None ) Label for the X-axis values. If y is a pandas Series, and ylabel is None , the label is inferred automatically. figsize : tuple (default: (5, 5) ) Matplotlib figure size. Returns plot : Matplotlib Figure object","title":"API"},{"location":"user_guide/plotting/scatterplotmatrix/","text":"scatterplotmatrix: visualize datasets via a scatter plot matrix A function for creating as scatter plot matrix in matplotlib from mlxtend.plotting import scatterplotmatrix Overview A matplotlib convenience function for creating a scatterplot matrix. References - Example 1 - Simple Scatter Plot Matrix import matplotlib.pyplot as plt from mlxtend.data import iris_data from mlxtend.plotting import scatterplotmatrix X, y = iris_data() scatterplotmatrix(X, figsize=(10, 8)) plt.tight_layout() plt.show() Example 2 - Scatter Plot Matrix with Multiple Categories names = ['sepal length [cm]', 'sepal width [cm]', 'petal length [cm]', 'petal width [cm]'] fig, axes = scatterplotmatrix(X[y==0], figsize=(10, 8), alpha=0.5) fig, axes = scatterplotmatrix(X[y==1], fig_axes=(fig, axes), alpha=0.5) fig, axes = scatterplotmatrix(X[y==2], fig_axes=(fig, axes), alpha=0.5, names=names) plt.tight_layout() plt.show() API scatterplotmatrix(X, fig_axes=None, names=None, figsize=(8, 8), alpha=1.0, kwargs) Lower triangular of a scatterplot matrix Parameters X : array-like, shape={num_examples, num_features} Design matrix containing data instances (examples) with multiple exploratory variables (features). fix_axes : tuple (default: None) A (fig, axes) tuple, where fig is an figure object and axes is an axes object created via matplotlib, for example, by calling the pyplot subplot function fig, axes = plt.subplots(...) names : list (default: None) A list of string names, which should have the same number of elements as there are features (columns) in X . figsize : tuple (default: (8, 8)) Height and width of the subplot grid. Ignored if fig_axes is not None . alpha : float (default: 1.0) Transparency for both the scatter plots and the histograms along the diagonal. **kwargs : kwargs Keyword arguments for the scatterplots. Returns fix_axes : tuple A (fig, axes) tuple, where fig is an figure object and axes is an axes object created via matplotlib, for example, by calling the pyplot subplot function fig, axes = plt.subplots(...)","title":"scatterplotmatrix: visualize datasets via a scatter plot matrix"},{"location":"user_guide/plotting/scatterplotmatrix/#scatterplotmatrix-visualize-datasets-via-a-scatter-plot-matrix","text":"A function for creating as scatter plot matrix in matplotlib from mlxtend.plotting import scatterplotmatrix","title":"scatterplotmatrix: visualize datasets via a scatter plot matrix"},{"location":"user_guide/plotting/scatterplotmatrix/#overview","text":"A matplotlib convenience function for creating a scatterplot matrix.","title":"Overview"},{"location":"user_guide/plotting/scatterplotmatrix/#references","text":"-","title":"References"},{"location":"user_guide/plotting/scatterplotmatrix/#example-1-simple-scatter-plot-matrix","text":"import matplotlib.pyplot as plt from mlxtend.data import iris_data from mlxtend.plotting import scatterplotmatrix X, y = iris_data() scatterplotmatrix(X, figsize=(10, 8)) plt.tight_layout() plt.show()","title":"Example 1 - Simple Scatter Plot Matrix"},{"location":"user_guide/plotting/scatterplotmatrix/#example-2-scatter-plot-matrix-with-multiple-categories","text":"names = ['sepal length [cm]', 'sepal width [cm]', 'petal length [cm]', 'petal width [cm]'] fig, axes = scatterplotmatrix(X[y==0], figsize=(10, 8), alpha=0.5) fig, axes = scatterplotmatrix(X[y==1], fig_axes=(fig, axes), alpha=0.5) fig, axes = scatterplotmatrix(X[y==2], fig_axes=(fig, axes), alpha=0.5, names=names) plt.tight_layout() plt.show()","title":"Example 2 - Scatter Plot Matrix with Multiple Categories"},{"location":"user_guide/plotting/scatterplotmatrix/#api","text":"scatterplotmatrix(X, fig_axes=None, names=None, figsize=(8, 8), alpha=1.0, kwargs) Lower triangular of a scatterplot matrix Parameters X : array-like, shape={num_examples, num_features} Design matrix containing data instances (examples) with multiple exploratory variables (features). fix_axes : tuple (default: None) A (fig, axes) tuple, where fig is an figure object and axes is an axes object created via matplotlib, for example, by calling the pyplot subplot function fig, axes = plt.subplots(...) names : list (default: None) A list of string names, which should have the same number of elements as there are features (columns) in X . figsize : tuple (default: (8, 8)) Height and width of the subplot grid. Ignored if fig_axes is not None . alpha : float (default: 1.0) Transparency for both the scatter plots and the histograms along the diagonal. **kwargs : kwargs Keyword arguments for the scatterplots. Returns fix_axes : tuple A (fig, axes) tuple, where fig is an figure object and axes is an axes object created via matplotlib, for example, by calling the pyplot subplot function fig, axes = plt.subplots(...)","title":"API"},{"location":"user_guide/plotting/stacked_barplot/","text":"stacked_barplot: Plot stacked bar plots in matplotlib A function to conveniently plot stacked bar plots in matplotlib using pandas DataFrame s. from mlxtend.plotting import stacked_barplot Overview A matplotlib convenience function for creating barplots from DataFrames where each sample is associated with several categories. References - Example 1 - Stacked Barplot from Pandas DataFrames import pandas as pd s1 = [1.0, 2.0, 3.0, 4.0] s2 = [1.4, 2.1, 2.9, 5.1] s3 = [1.9, 2.2, 3.5, 4.1] s4 = [1.4, 2.5, 3.5, 4.2] data = [s1, s2, s3, s4] df = pd.DataFrame(data, columns=['X1', 'X2', 'X3', 'X4']) df.columns = ['X1', 'X2', 'X3', 'X4'] df.index = ['Sample1', 'Sample2', 'Sample3', 'Sample4'] df X1 X2 X3 X4 Sample1 1.0 2.0 3.0 4.0 Sample2 1.4 2.1 2.9 5.1 Sample3 1.9 2.2 3.5 4.1 Sample4 1.4 2.5 3.5 4.2 By default, the index of the DataFrame is used as column labels, and the DataFrame columns are used for the plot legend. import matplotlib.pyplot as plt from mlxtend.plotting import stacked_barplot fig = stacked_barplot(df, rotation=45, legend_loc='best') API stacked_barplot(df, bar_width='auto', colors='bgrcky', labels='index', rotation=90, legend_loc='best') Function to plot stacked barplots Parameters df : pandas.DataFrame A pandas DataFrame where the index denotes the x-axis labels, and the columns contain the different measurements for each row. bar_width: 'auto' or float (default: 'auto') Parameter to set the widths of the bars. if 'auto', the width is automatically determined by the number of columns in the dataset. colors: str (default: 'bgrcky') The colors of the bars. labels: 'index' or iterable (default: 'index') If 'index', the DataFrame index will be used as x-tick labels. rotation: int (default: 90) Parameter to rotate the x-axis labels. legend_loc : str (default: 'best') Location of the plot legend {best, upper left, upper right, lower left, lower right} No legend if legend_loc=False Returns fig : matplotlib.pyplot figure object Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/plotting/stacked_barplot/","title":"stacked_barplot: Plot stacked bar plots in matplotlib"},{"location":"user_guide/plotting/stacked_barplot/#stacked_barplot-plot-stacked-bar-plots-in-matplotlib","text":"A function to conveniently plot stacked bar plots in matplotlib using pandas DataFrame s. from mlxtend.plotting import stacked_barplot","title":"stacked_barplot: Plot stacked bar plots in matplotlib"},{"location":"user_guide/plotting/stacked_barplot/#overview","text":"A matplotlib convenience function for creating barplots from DataFrames where each sample is associated with several categories.","title":"Overview"},{"location":"user_guide/plotting/stacked_barplot/#references","text":"-","title":"References"},{"location":"user_guide/plotting/stacked_barplot/#example-1-stacked-barplot-from-pandas-dataframes","text":"import pandas as pd s1 = [1.0, 2.0, 3.0, 4.0] s2 = [1.4, 2.1, 2.9, 5.1] s3 = [1.9, 2.2, 3.5, 4.1] s4 = [1.4, 2.5, 3.5, 4.2] data = [s1, s2, s3, s4] df = pd.DataFrame(data, columns=['X1', 'X2', 'X3', 'X4']) df.columns = ['X1', 'X2', 'X3', 'X4'] df.index = ['Sample1', 'Sample2', 'Sample3', 'Sample4'] df X1 X2 X3 X4 Sample1 1.0 2.0 3.0 4.0 Sample2 1.4 2.1 2.9 5.1 Sample3 1.9 2.2 3.5 4.1 Sample4 1.4 2.5 3.5 4.2 By default, the index of the DataFrame is used as column labels, and the DataFrame columns are used for the plot legend. import matplotlib.pyplot as plt from mlxtend.plotting import stacked_barplot fig = stacked_barplot(df, rotation=45, legend_loc='best')","title":"Example 1 - Stacked Barplot from Pandas DataFrames"},{"location":"user_guide/plotting/stacked_barplot/#api","text":"stacked_barplot(df, bar_width='auto', colors='bgrcky', labels='index', rotation=90, legend_loc='best') Function to plot stacked barplots Parameters df : pandas.DataFrame A pandas DataFrame where the index denotes the x-axis labels, and the columns contain the different measurements for each row. bar_width: 'auto' or float (default: 'auto') Parameter to set the widths of the bars. if 'auto', the width is automatically determined by the number of columns in the dataset. colors: str (default: 'bgrcky') The colors of the bars. labels: 'index' or iterable (default: 'index') If 'index', the DataFrame index will be used as x-tick labels. rotation: int (default: 90) Parameter to rotate the x-axis labels. legend_loc : str (default: 'best') Location of the plot legend {best, upper left, upper right, lower left, lower right} No legend if legend_loc=False Returns fig : matplotlib.pyplot figure object Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/plotting/stacked_barplot/","title":"API"},{"location":"user_guide/preprocessing/CopyTransformer/","text":"CopyTransformer: A function that creates a copy of the input array in a scikit-learn pipeline A simple transformer that returns a copy of the input array, for example, as part of a scikit-learn pipeline. from mlxtend.preprocessing import CopyTransformer Example 1 from sklearn.pipeline import Pipeline from sklearn.model_selection import GridSearchCV from sklearn.ensemble import RandomForestClassifier from sklearn.feature_extraction.text import CountVectorizer from mlxtend.preprocessing import CopyTransformer import re import numpy as np X_train = np.array(['abc def ghi', 'this is a test', 'this is a test', 'this is a test']) y_train = np.array([0, 0, 1, 1]) pipe_1 = Pipeline([ ('vect', CountVectorizer()), ('to_dense', CopyTransformer()), ('clf', RandomForestClassifier()) ]) parameters_1 = dict( clf__n_estimators=[50, 100, 200], clf__max_features=['sqrt', 'log2', None],) grid_search_1 = GridSearchCV(pipe_1, parameters_1, n_jobs=1, verbose=1, scoring='accuracy', cv=2) print(\"Performing grid search...\") print(\"pipeline:\", [name for name, _ in pipe_1.steps]) print(\"parameters:\") grid_search_1.fit(X_train, y_train) print(\"Best score: %0.3f\" % grid_search_1.best_score_) print(\"Best parameters set:\") best_parameters_1 = grid_search_1.best_estimator_.get_params() for param_name in sorted(parameters_1.keys()): print(\"\\t%s: %r\" % (param_name, best_parameters_1[param_name])) Performing grid search... pipeline: ['vect', 'to_dense', 'clf'] parameters: Fitting 2 folds for each of 9 candidates, totalling 18 fits Best score: 0.500 Best parameters set: clf__max_features: 'sqrt' clf__n_estimators: 50 [Parallel(n_jobs=1)]: Done 18 out of 18 | elapsed: 2.9s finished API CopyTransformer() Transformer that returns a copy of the input array For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/preprocessing/CopyTransformer/ Methods fit(X, y=None) Mock method. Does nothing. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] (default: None) Returns self fit_transform(X, y=None) Return a copy of the input array. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] (default: None) Returns X_copy : copy of the input X array. get_params(deep=True) Get parameters for this estimator. Parameters deep : boolean, optional If True, will return the parameters for this estimator and contained subobjects that are estimators. Returns params : mapping of string to any Parameter names mapped to their values. set_params( params) Set the parameters of this estimator. The method works on simple estimators as well as on nested objects (such as pipelines). The latter have parameters of the form <component>__<parameter> so that it's possible to update each component of a nested object. Returns self transform(X, y=None) Return a copy of the input array. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] (default: None) Returns X_copy : copy of the input X array.","title":"CopyTransformer: A function that creates a copy of the input array in a scikit-learn pipeline"},{"location":"user_guide/preprocessing/CopyTransformer/#copytransformer-a-function-that-creates-a-copy-of-the-input-array-in-a-scikit-learn-pipeline","text":"A simple transformer that returns a copy of the input array, for example, as part of a scikit-learn pipeline. from mlxtend.preprocessing import CopyTransformer","title":"CopyTransformer: A function that creates a copy of the input array in a scikit-learn pipeline"},{"location":"user_guide/preprocessing/CopyTransformer/#example-1","text":"from sklearn.pipeline import Pipeline from sklearn.model_selection import GridSearchCV from sklearn.ensemble import RandomForestClassifier from sklearn.feature_extraction.text import CountVectorizer from mlxtend.preprocessing import CopyTransformer import re import numpy as np X_train = np.array(['abc def ghi', 'this is a test', 'this is a test', 'this is a test']) y_train = np.array([0, 0, 1, 1]) pipe_1 = Pipeline([ ('vect', CountVectorizer()), ('to_dense', CopyTransformer()), ('clf', RandomForestClassifier()) ]) parameters_1 = dict( clf__n_estimators=[50, 100, 200], clf__max_features=['sqrt', 'log2', None],) grid_search_1 = GridSearchCV(pipe_1, parameters_1, n_jobs=1, verbose=1, scoring='accuracy', cv=2) print(\"Performing grid search...\") print(\"pipeline:\", [name for name, _ in pipe_1.steps]) print(\"parameters:\") grid_search_1.fit(X_train, y_train) print(\"Best score: %0.3f\" % grid_search_1.best_score_) print(\"Best parameters set:\") best_parameters_1 = grid_search_1.best_estimator_.get_params() for param_name in sorted(parameters_1.keys()): print(\"\\t%s: %r\" % (param_name, best_parameters_1[param_name])) Performing grid search... pipeline: ['vect', 'to_dense', 'clf'] parameters: Fitting 2 folds for each of 9 candidates, totalling 18 fits Best score: 0.500 Best parameters set: clf__max_features: 'sqrt' clf__n_estimators: 50 [Parallel(n_jobs=1)]: Done 18 out of 18 | elapsed: 2.9s finished","title":"Example 1"},{"location":"user_guide/preprocessing/CopyTransformer/#api","text":"CopyTransformer() Transformer that returns a copy of the input array For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/preprocessing/CopyTransformer/","title":"API"},{"location":"user_guide/preprocessing/CopyTransformer/#methods","text":"fit(X, y=None) Mock method. Does nothing. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] (default: None) Returns self fit_transform(X, y=None) Return a copy of the input array. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] (default: None) Returns X_copy : copy of the input X array. get_params(deep=True) Get parameters for this estimator. Parameters deep : boolean, optional If True, will return the parameters for this estimator and contained subobjects that are estimators. Returns params : mapping of string to any Parameter names mapped to their values. set_params( params) Set the parameters of this estimator. The method works on simple estimators as well as on nested objects (such as pipelines). The latter have parameters of the form <component>__<parameter> so that it's possible to update each component of a nested object. Returns self transform(X, y=None) Return a copy of the input array. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] (default: None) Returns X_copy : copy of the input X array.","title":"Methods"},{"location":"user_guide/preprocessing/DenseTransformer/","text":"DenseTransformer: Transforms a sparse into a dense NumPy array, e.g., in a scikit-learn pipeline A simple transformer that converts a sparse into a dense numpy array, e.g., required for scikit-learn's Pipeline when, for example, CountVectorizers are used in combination with estimators that are not compatible with sparse matrices. from mlxtend.preprocessing import DenseTransformer Example 1 from sklearn.pipeline import Pipeline from sklearn.model_selection import GridSearchCV from sklearn.ensemble import RandomForestClassifier from sklearn.feature_extraction.text import CountVectorizer from mlxtend.preprocessing import DenseTransformer import re import numpy as np X_train = np.array(['abc def ghi', 'this is a test', 'this is a test', 'this is a test']) y_train = np.array([0, 0, 1, 1]) pipe_1 = Pipeline([ ('vect', CountVectorizer()), ('to_dense', DenseTransformer()), ('clf', RandomForestClassifier()) ]) parameters_1 = dict( clf__n_estimators=[50, 100, 200], clf__max_features=['sqrt', 'log2', None],) grid_search_1 = GridSearchCV(pipe_1, parameters_1, n_jobs=1, verbose=1, scoring='accuracy', cv=2) print(\"Performing grid search...\") print(\"pipeline:\", [name for name, _ in pipe_1.steps]) print(\"parameters:\") grid_search_1.fit(X_train, y_train) print(\"Best score: %0.3f\" % grid_search_1.best_score_) print(\"Best parameters set:\") best_parameters_1 = grid_search_1.best_estimator_.get_params() for param_name in sorted(parameters_1.keys()): print(\"\\t%s: %r\" % (param_name, best_parameters_1[param_name])) Performing grid search... pipeline: ['vect', 'to_dense', 'clf'] parameters: Fitting 2 folds for each of 9 candidates, totalling 18 fits Best score: 0.500 Best parameters set: clf__max_features: 'sqrt' clf__n_estimators: 50 [Parallel(n_jobs=1)]: Done 18 out of 18 | elapsed: 3.9s finished API DenseTransformer(return_copy=True) Convert a sparse array into a dense array. For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/preprocessing/DenseTransformer/ Methods fit(X, y=None) Mock method. Does nothing. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] (default: None) Returns self fit_transform(X, y=None) Return a dense version of the input array. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] (default: None) Returns X_dense : dense version of the input X array. get_params(deep=True) Get parameters for this estimator. Parameters deep : boolean, optional If True, will return the parameters for this estimator and contained subobjects that are estimators. Returns params : mapping of string to any Parameter names mapped to their values. set_params( params) Set the parameters of this estimator. The method works on simple estimators as well as on nested objects (such as pipelines). The latter have parameters of the form <component>__<parameter> so that it's possible to update each component of a nested object. Returns self transform(X, y=None) Return a dense version of the input array. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] (default: None) Returns X_dense : dense version of the input X array.","title":"DenseTransformer: Transforms a sparse into a dense NumPy array, e.g., in a scikit-learn pipeline"},{"location":"user_guide/preprocessing/DenseTransformer/#densetransformer-transforms-a-sparse-into-a-dense-numpy-array-eg-in-a-scikit-learn-pipeline","text":"A simple transformer that converts a sparse into a dense numpy array, e.g., required for scikit-learn's Pipeline when, for example, CountVectorizers are used in combination with estimators that are not compatible with sparse matrices. from mlxtend.preprocessing import DenseTransformer","title":"DenseTransformer: Transforms a sparse into a dense NumPy array, e.g., in a scikit-learn pipeline"},{"location":"user_guide/preprocessing/DenseTransformer/#example-1","text":"from sklearn.pipeline import Pipeline from sklearn.model_selection import GridSearchCV from sklearn.ensemble import RandomForestClassifier from sklearn.feature_extraction.text import CountVectorizer from mlxtend.preprocessing import DenseTransformer import re import numpy as np X_train = np.array(['abc def ghi', 'this is a test', 'this is a test', 'this is a test']) y_train = np.array([0, 0, 1, 1]) pipe_1 = Pipeline([ ('vect', CountVectorizer()), ('to_dense', DenseTransformer()), ('clf', RandomForestClassifier()) ]) parameters_1 = dict( clf__n_estimators=[50, 100, 200], clf__max_features=['sqrt', 'log2', None],) grid_search_1 = GridSearchCV(pipe_1, parameters_1, n_jobs=1, verbose=1, scoring='accuracy', cv=2) print(\"Performing grid search...\") print(\"pipeline:\", [name for name, _ in pipe_1.steps]) print(\"parameters:\") grid_search_1.fit(X_train, y_train) print(\"Best score: %0.3f\" % grid_search_1.best_score_) print(\"Best parameters set:\") best_parameters_1 = grid_search_1.best_estimator_.get_params() for param_name in sorted(parameters_1.keys()): print(\"\\t%s: %r\" % (param_name, best_parameters_1[param_name])) Performing grid search... pipeline: ['vect', 'to_dense', 'clf'] parameters: Fitting 2 folds for each of 9 candidates, totalling 18 fits Best score: 0.500 Best parameters set: clf__max_features: 'sqrt' clf__n_estimators: 50 [Parallel(n_jobs=1)]: Done 18 out of 18 | elapsed: 3.9s finished","title":"Example 1"},{"location":"user_guide/preprocessing/DenseTransformer/#api","text":"DenseTransformer(return_copy=True) Convert a sparse array into a dense array. For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/preprocessing/DenseTransformer/","title":"API"},{"location":"user_guide/preprocessing/DenseTransformer/#methods","text":"fit(X, y=None) Mock method. Does nothing. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] (default: None) Returns self fit_transform(X, y=None) Return a dense version of the input array. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] (default: None) Returns X_dense : dense version of the input X array. get_params(deep=True) Get parameters for this estimator. Parameters deep : boolean, optional If True, will return the parameters for this estimator and contained subobjects that are estimators. Returns params : mapping of string to any Parameter names mapped to their values. set_params( params) Set the parameters of this estimator. The method works on simple estimators as well as on nested objects (such as pipelines). The latter have parameters of the form <component>__<parameter> so that it's possible to update each component of a nested object. Returns self transform(X, y=None) Return a dense version of the input array. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] (default: None) Returns X_dense : dense version of the input X array.","title":"Methods"},{"location":"user_guide/preprocessing/MeanCenterer/","text":"MeanCenterer: column-based mean centering on a NumPy array A transformer object that performs column-based mean centering on a NumPy array. from mlxtend.preprocessing import MeanCenterer Example 1 - Centering a NumPy Array Use the fit method to fit the column means of a dataset (e.g., the training dataset) to a new MeanCenterer object. Then, call the transform method on the same dataset to center it at the sample mean. import numpy as np from mlxtend.preprocessing import MeanCenterer X_train = np.array( [[1, 2, 3], [4, 5, 6], [7, 8, 9]]) mc = MeanCenterer().fit(X_train) mc.transform(X_train) array([[-3., -3., -3.], [ 0., 0., 0.], [ 3., 3., 3.]]) API MeanCenterer() Column centering of vectors and matrices. Attributes col_means : numpy.ndarray [n_columns] NumPy array storing the mean values for centering after fitting the MeanCenterer object. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/preprocessing/MeanCenterer/ Methods fit(X) Gets the column means for mean centering. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Array of data vectors, where n_samples is the number of samples and n_features is the number of features. Returns self fit_transform(X) Fits and transforms an arry. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Array of data vectors, where n_samples is the number of samples and n_features is the number of features. Returns X_tr : {array-like, sparse matrix}, shape = [n_samples, n_features] A copy of the input array with the columns centered. transform(X) Centers a NumPy array. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Array of data vectors, where n_samples is the number of samples and n_features is the number of features. Returns X_tr : {array-like, sparse matrix}, shape = [n_samples, n_features] A copy of the input array with the columns centered. ython","title":"MeanCenterer: column-based mean centering on a NumPy array"},{"location":"user_guide/preprocessing/MeanCenterer/#meancenterer-column-based-mean-centering-on-a-numpy-array","text":"A transformer object that performs column-based mean centering on a NumPy array. from mlxtend.preprocessing import MeanCenterer","title":"MeanCenterer: column-based mean centering on a NumPy array"},{"location":"user_guide/preprocessing/MeanCenterer/#example-1-centering-a-numpy-array","text":"Use the fit method to fit the column means of a dataset (e.g., the training dataset) to a new MeanCenterer object. Then, call the transform method on the same dataset to center it at the sample mean. import numpy as np from mlxtend.preprocessing import MeanCenterer X_train = np.array( [[1, 2, 3], [4, 5, 6], [7, 8, 9]]) mc = MeanCenterer().fit(X_train) mc.transform(X_train) array([[-3., -3., -3.], [ 0., 0., 0.], [ 3., 3., 3.]])","title":"Example 1 - Centering a NumPy Array"},{"location":"user_guide/preprocessing/MeanCenterer/#api","text":"MeanCenterer() Column centering of vectors and matrices. Attributes col_means : numpy.ndarray [n_columns] NumPy array storing the mean values for centering after fitting the MeanCenterer object. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/preprocessing/MeanCenterer/","title":"API"},{"location":"user_guide/preprocessing/MeanCenterer/#methods","text":"fit(X) Gets the column means for mean centering. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Array of data vectors, where n_samples is the number of samples and n_features is the number of features. Returns self fit_transform(X) Fits and transforms an arry. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Array of data vectors, where n_samples is the number of samples and n_features is the number of features. Returns X_tr : {array-like, sparse matrix}, shape = [n_samples, n_features] A copy of the input array with the columns centered. transform(X) Centers a NumPy array. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Array of data vectors, where n_samples is the number of samples and n_features is the number of features. Returns X_tr : {array-like, sparse matrix}, shape = [n_samples, n_features] A copy of the input array with the columns centered. ython","title":"Methods"},{"location":"user_guide/preprocessing/TransactionEncoder/","text":"TransactionEncoder: Convert item lists into transaction data for frequent itemset mining Encoder class for transaction data in Python lists from mlxtend.preprocessing import TransactionEncoder Overview Encodes database transaction data in form of a Python list of lists into a NumPy array. Example 1 Suppose we have the following transaction data: from mlxtend.preprocessing import TransactionEncoder dataset = [['Apple', 'Beer', 'Rice', 'Chicken'], ['Apple', 'Beer', 'Rice'], ['Apple', 'Beer'], ['Apple', 'Bananas'], ['Milk', 'Beer', 'Rice', 'Chicken'], ['Milk', 'Beer', 'Rice'], ['Milk', 'Beer'], ['Apple', 'Bananas']] Using and TransactionEncoder object, we can transform this dataset into an array format suitable for typical machine learning APIs. Via the fit method, the TransactionEncoder learns the unique labels in the dataset, and via the transform method, it transforms the input dataset (a Python list of lists) into a one-hot encoded NumPy boolean array: te = TransactionEncoder() te_ary = te.fit(dataset).transform(dataset) te_ary array([[ True, False, True, True, False, True], [ True, False, True, False, False, True], [ True, False, True, False, False, False], [ True, True, False, False, False, False], [False, False, True, True, True, True], [False, False, True, False, True, True], [False, False, True, False, True, False], [ True, True, False, False, False, False]], dtype=bool) The NumPy array is boolean for the sake of memory efficiency when working with large datasets. If a classic integer representation is desired instead, we can just convert the array to the appropriate type: te_ary.astype(\"int\") array([[1, 0, 1, 1, 0, 1], [1, 0, 1, 0, 0, 1], [1, 0, 1, 0, 0, 0], [1, 1, 0, 0, 0, 0], [0, 0, 1, 1, 1, 1], [0, 0, 1, 0, 1, 1], [0, 0, 1, 0, 1, 0], [1, 1, 0, 0, 0, 0]]) After fitting, the unique column names that correspond to the data array shown above can be accessed via the columns_ attribute: te.columns_ ['Apple', 'Bananas', 'Beer', 'Chicken', 'Milk', 'Rice'] For our convenience, we can turn theencoded array into a pandas DataFrame : import pandas as pd pd.DataFrame(te_ary, columns=te.columns_) .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } Apple Bananas Beer Chicken Milk Rice 0 True False True True False True 1 True False True False False True 2 True False True False False False 3 True True False False False False 4 False False True True True True 5 False False True False True True 6 False False True False True False 7 True True False False False False If we desire, we can turn the one-hot encoded array back into a transaction list of lists via the inverse_transform function: first4 = te_ary[:4] te.inverse_transform(first4) [['Apple', 'Beer', 'Chicken', 'Rice'], ['Apple', 'Beer', 'Rice'], ['Apple', 'Beer'], ['Apple', 'Bananas']] API TransactionEncoder() Encoder class for transaction data in Python lists Parameters None Attributes columns_: list List of unique names in the X input list of lists Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/preprocessing/TransactionEncoder/ Methods fit(X) Learn unique column names from transaction DataFrame Parameters X : list of lists A python list of lists, where the outer list stores the n transactions and the inner list stores the items in each transaction. For example, [['Apple', 'Beer', 'Rice', 'Chicken'], ['Apple', 'Beer', 'Rice'], ['Apple', 'Beer'], ['Apple', 'Bananas'], ['Milk', 'Beer', 'Rice', 'Chicken'], ['Milk', 'Beer', 'Rice'], ['Milk', 'Beer'], ['Apple', 'Bananas']] fit_transform(X, sparse=False) Fit a TransactionEncoder encoder and transform a dataset. get_params(deep=True) Get parameters for this estimator. Parameters deep : boolean, optional If True, will return the parameters for this estimator and contained subobjects that are estimators. Returns params : mapping of string to any Parameter names mapped to their values. inverse_transform(array) Transforms an encoded NumPy array back into transactions. Parameters array : NumPy array [n_transactions, n_unique_items] The NumPy one-hot encoded boolean array of the input transactions, where the columns represent the unique items found in the input array in alphabetic order For example, array([[True , False, True , True , False, True ], [True , False, True , False, False, True ], [True , False, True , False, False, False], [True , True , False, False, False, False], [False, False, True , True , True , True ], [False, False, True , False, True , True ], [False, False, True , False, True , False], [True , True , False, False, False, False]]) The corresponding column labels are available as self.columns_, e.g., ['Apple', 'Bananas', 'Beer', 'Chicken', 'Milk', 'Rice'] Returns X : list of lists A python list of lists, where the outer list stores the n transactions and the inner list stores the items in each transaction. For example, [['Apple', 'Beer', 'Rice', 'Chicken'], ['Apple', 'Beer', 'Rice'], ['Apple', 'Beer'], ['Apple', 'Bananas'], ['Milk', 'Beer', 'Rice', 'Chicken'], ['Milk', 'Beer', 'Rice'], ['Milk', 'Beer'], ['Apple', 'Bananas']] set_params( params) Set the parameters of this estimator. The method works on simple estimators as well as on nested objects (such as pipelines). The latter have parameters of the form <component>__<parameter> so that it's possible to update each component of a nested object. Returns self transform(X, sparse=False) Transform transactions into a one-hot encoded NumPy array. Parameters X : list of lists A python list of lists, where the outer list stores the n transactions and the inner list stores the items in each transaction. For example, [['Apple', 'Beer', 'Rice', 'Chicken'], ['Apple', 'Beer', 'Rice'], ['Apple', 'Beer'], ['Apple', 'Bananas'], ['Milk', 'Beer', 'Rice', 'Chicken'], ['Milk', 'Beer', 'Rice'], ['Milk', 'Beer'], ['Apple', 'Bananas']] sparse: bool (default=False) If True, transform will return Compressed Sparse Row matrix instead of the regular one. Returns array : NumPy array [n_transactions, n_unique_items] if sparse=False (default). Compressed Sparse Row matrix otherwise The one-hot encoded boolean array of the input transactions, where the columns represent the unique items found in the input array in alphabetic order. Exact representation depends on the sparse argument For example, array([[True , False, True , True , False, True ], [True , False, True , False, False, True ], [True , False, True , False, False, False], [True , True , False, False, False, False], [False, False, True , True , True , True ], [False, False, True , False, True , True ], [False, False, True , False, True , False], [True , True , False, False, False, False]]) The corresponding column labels are available as self.columns_, e.g., ['Apple', 'Bananas', 'Beer', 'Chicken', 'Milk', 'Rice'] ython","title":"TransactionEncoder"},{"location":"user_guide/preprocessing/TransactionEncoder/#transactionencoder-convert-item-lists-into-transaction-data-for-frequent-itemset-mining","text":"Encoder class for transaction data in Python lists from mlxtend.preprocessing import TransactionEncoder","title":"TransactionEncoder: Convert item lists into transaction data for frequent itemset mining"},{"location":"user_guide/preprocessing/TransactionEncoder/#overview","text":"Encodes database transaction data in form of a Python list of lists into a NumPy array.","title":"Overview"},{"location":"user_guide/preprocessing/TransactionEncoder/#example-1","text":"Suppose we have the following transaction data: from mlxtend.preprocessing import TransactionEncoder dataset = [['Apple', 'Beer', 'Rice', 'Chicken'], ['Apple', 'Beer', 'Rice'], ['Apple', 'Beer'], ['Apple', 'Bananas'], ['Milk', 'Beer', 'Rice', 'Chicken'], ['Milk', 'Beer', 'Rice'], ['Milk', 'Beer'], ['Apple', 'Bananas']] Using and TransactionEncoder object, we can transform this dataset into an array format suitable for typical machine learning APIs. Via the fit method, the TransactionEncoder learns the unique labels in the dataset, and via the transform method, it transforms the input dataset (a Python list of lists) into a one-hot encoded NumPy boolean array: te = TransactionEncoder() te_ary = te.fit(dataset).transform(dataset) te_ary array([[ True, False, True, True, False, True], [ True, False, True, False, False, True], [ True, False, True, False, False, False], [ True, True, False, False, False, False], [False, False, True, True, True, True], [False, False, True, False, True, True], [False, False, True, False, True, False], [ True, True, False, False, False, False]], dtype=bool) The NumPy array is boolean for the sake of memory efficiency when working with large datasets. If a classic integer representation is desired instead, we can just convert the array to the appropriate type: te_ary.astype(\"int\") array([[1, 0, 1, 1, 0, 1], [1, 0, 1, 0, 0, 1], [1, 0, 1, 0, 0, 0], [1, 1, 0, 0, 0, 0], [0, 0, 1, 1, 1, 1], [0, 0, 1, 0, 1, 1], [0, 0, 1, 0, 1, 0], [1, 1, 0, 0, 0, 0]]) After fitting, the unique column names that correspond to the data array shown above can be accessed via the columns_ attribute: te.columns_ ['Apple', 'Bananas', 'Beer', 'Chicken', 'Milk', 'Rice'] For our convenience, we can turn theencoded array into a pandas DataFrame : import pandas as pd pd.DataFrame(te_ary, columns=te.columns_) .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } Apple Bananas Beer Chicken Milk Rice 0 True False True True False True 1 True False True False False True 2 True False True False False False 3 True True False False False False 4 False False True True True True 5 False False True False True True 6 False False True False True False 7 True True False False False False If we desire, we can turn the one-hot encoded array back into a transaction list of lists via the inverse_transform function: first4 = te_ary[:4] te.inverse_transform(first4) [['Apple', 'Beer', 'Chicken', 'Rice'], ['Apple', 'Beer', 'Rice'], ['Apple', 'Beer'], ['Apple', 'Bananas']]","title":"Example 1"},{"location":"user_guide/preprocessing/TransactionEncoder/#api","text":"TransactionEncoder() Encoder class for transaction data in Python lists Parameters None Attributes columns_: list List of unique names in the X input list of lists Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/preprocessing/TransactionEncoder/","title":"API"},{"location":"user_guide/preprocessing/TransactionEncoder/#methods","text":"fit(X) Learn unique column names from transaction DataFrame Parameters X : list of lists A python list of lists, where the outer list stores the n transactions and the inner list stores the items in each transaction. For example, [['Apple', 'Beer', 'Rice', 'Chicken'], ['Apple', 'Beer', 'Rice'], ['Apple', 'Beer'], ['Apple', 'Bananas'], ['Milk', 'Beer', 'Rice', 'Chicken'], ['Milk', 'Beer', 'Rice'], ['Milk', 'Beer'], ['Apple', 'Bananas']] fit_transform(X, sparse=False) Fit a TransactionEncoder encoder and transform a dataset. get_params(deep=True) Get parameters for this estimator. Parameters deep : boolean, optional If True, will return the parameters for this estimator and contained subobjects that are estimators. Returns params : mapping of string to any Parameter names mapped to their values. inverse_transform(array) Transforms an encoded NumPy array back into transactions. Parameters array : NumPy array [n_transactions, n_unique_items] The NumPy one-hot encoded boolean array of the input transactions, where the columns represent the unique items found in the input array in alphabetic order For example, array([[True , False, True , True , False, True ], [True , False, True , False, False, True ], [True , False, True , False, False, False], [True , True , False, False, False, False], [False, False, True , True , True , True ], [False, False, True , False, True , True ], [False, False, True , False, True , False], [True , True , False, False, False, False]]) The corresponding column labels are available as self.columns_, e.g., ['Apple', 'Bananas', 'Beer', 'Chicken', 'Milk', 'Rice'] Returns X : list of lists A python list of lists, where the outer list stores the n transactions and the inner list stores the items in each transaction. For example, [['Apple', 'Beer', 'Rice', 'Chicken'], ['Apple', 'Beer', 'Rice'], ['Apple', 'Beer'], ['Apple', 'Bananas'], ['Milk', 'Beer', 'Rice', 'Chicken'], ['Milk', 'Beer', 'Rice'], ['Milk', 'Beer'], ['Apple', 'Bananas']] set_params( params) Set the parameters of this estimator. The method works on simple estimators as well as on nested objects (such as pipelines). The latter have parameters of the form <component>__<parameter> so that it's possible to update each component of a nested object. Returns self transform(X, sparse=False) Transform transactions into a one-hot encoded NumPy array. Parameters X : list of lists A python list of lists, where the outer list stores the n transactions and the inner list stores the items in each transaction. For example, [['Apple', 'Beer', 'Rice', 'Chicken'], ['Apple', 'Beer', 'Rice'], ['Apple', 'Beer'], ['Apple', 'Bananas'], ['Milk', 'Beer', 'Rice', 'Chicken'], ['Milk', 'Beer', 'Rice'], ['Milk', 'Beer'], ['Apple', 'Bananas']] sparse: bool (default=False) If True, transform will return Compressed Sparse Row matrix instead of the regular one. Returns array : NumPy array [n_transactions, n_unique_items] if sparse=False (default). Compressed Sparse Row matrix otherwise The one-hot encoded boolean array of the input transactions, where the columns represent the unique items found in the input array in alphabetic order. Exact representation depends on the sparse argument For example, array([[True , False, True , True , False, True ], [True , False, True , False, False, True ], [True , False, True , False, False, False], [True , True , False, False, False, False], [False, False, True , True , True , True ], [False, False, True , False, True , True ], [False, False, True , False, True , False], [True , True , False, False, False, False]]) The corresponding column labels are available as self.columns_, e.g., ['Apple', 'Bananas', 'Beer', 'Chicken', 'Milk', 'Rice'] ython","title":"Methods"},{"location":"user_guide/preprocessing/minmax_scaling/","text":"MinMaxScaling: Min-max scaling fpr pandas DataFrames and NumPy arrays A function for min-max scaling of pandas DataFrames or NumPy arrays. from mlxtend.preprocessing import MinMaxScaling An alternative approach to Z-score normalization (or standardization) is the so-called Min-Max scaling (often also simply called \"normalization\" - a common cause for ambiguities). In this approach, the data is scaled to a fixed range - usually 0 to 1. The cost of having this bounded range - in contrast to standardization - is that we will end up with smaller standard deviations, which can suppress the effect of outliers. A Min-Max scaling is typically done via the following equation: X_{sc} = \\frac{X - X_{min}}{X_{max} - X_{min}}. One family of algorithms that is scale-invariant encompasses tree-based learning algorithms. Let's take the general CART decision tree algorithm. Without going into much depth regarding information gain and impurity measures, we can think of the decision as \"is feature x_i >= some_val?\" Intuitively, we can see that it really doesn't matter on which scale this feature is (centimeters, Fahrenheit, a standardized scale -- it really doesn't matter). Some examples of algorithms where feature scaling matters are: k-nearest neighbors with an Euclidean distance measure if want all features to contribute equally k-means (see k-nearest neighbors) logistic regression, SVMs, perceptrons, neural networks etc. if you are using gradient descent/ascent-based optimization, otherwise some weights will update much faster than others linear discriminant analysis, principal component analysis, kernel principal component analysis since you want to find directions of maximizing the variance (under the constraints that those directions/eigenvectors/principal components are orthogonal); you want to have features on the same scale since you'd emphasize variables on \"larger measurement scales\" more. There are many more cases than I can possibly list here ... I always recommend you to think about the algorithm and what it's doing, and then it typically becomes obvious whether we want to scale your features or not. In addition, we'd also want to think about whether we want to \"standardize\" or \"normalize\" (here: scaling to [0, 1] range) our data. Some algorithms assume that our data is centered at 0. For example, if we initialize the weights of a small multi-layer perceptron with tanh activation units to 0 or small random values centered around zero, we want to update the model weights \"equally.\" As a rule of thumb I'd say: When in doubt, just standardize the data, it shouldn't hurt. Example 1 - Scaling a Pandas DataFrame import pandas as pd s1 = pd.Series([1, 2, 3, 4, 5, 6], index=(range(6))) s2 = pd.Series([10, 9, 8, 7, 6, 5], index=(range(6))) df = pd.DataFrame(s1, columns=['s1']) df['s2'] = s2 df .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } s1 s2 0 1 10 1 2 9 2 3 8 3 4 7 4 5 6 5 6 5 from mlxtend.preprocessing import minmax_scaling minmax_scaling(df, columns=['s1', 's2']) .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } s1 s2 0 0.0 1.0 1 0.2 0.8 2 0.4 0.6 3 0.6 0.4 4 0.8 0.2 5 1.0 0.0 Example 2 - Scaling a NumPy Array import numpy as np X = np.array([[1, 10], [2, 9], [3, 8], [4, 7], [5, 6], [6, 5]]) X array([[ 1, 10], [ 2, 9], [ 3, 8], [ 4, 7], [ 5, 6], [ 6, 5]]) from mlxtend.preprocessing import minmax_scaling minmax_scaling(X, columns=[0, 1]) array([[0. , 1. ], [0.2, 0.8], [0.4, 0.6], [0.6, 0.4], [0.8, 0.2], [1. , 0. ]]) API minmax_scaling(array, columns, min_val=0, max_val=1) Min max scaling of pandas' DataFrames. Parameters array : pandas DataFrame or NumPy ndarray, shape = [n_rows, n_columns]. columns : array-like, shape = [n_columns] Array-like with column names, e.g., ['col1', 'col2', ...] or column indices [0, 2, 4, ...] min_val : int or float , optional (default= 0 ) minimum value after rescaling. max_val : int or float , optional (default= 1 ) maximum value after rescaling. Returns df_new : pandas DataFrame object. Copy of the array or DataFrame with rescaled columns. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/preprocessing/minmax_scaling/","title":"MinMaxScaling: Min-max scaling fpr pandas DataFrames and NumPy arrays"},{"location":"user_guide/preprocessing/minmax_scaling/#minmaxscaling-min-max-scaling-fpr-pandas-dataframes-and-numpy-arrays","text":"A function for min-max scaling of pandas DataFrames or NumPy arrays. from mlxtend.preprocessing import MinMaxScaling An alternative approach to Z-score normalization (or standardization) is the so-called Min-Max scaling (often also simply called \"normalization\" - a common cause for ambiguities). In this approach, the data is scaled to a fixed range - usually 0 to 1. The cost of having this bounded range - in contrast to standardization - is that we will end up with smaller standard deviations, which can suppress the effect of outliers. A Min-Max scaling is typically done via the following equation: X_{sc} = \\frac{X - X_{min}}{X_{max} - X_{min}}. One family of algorithms that is scale-invariant encompasses tree-based learning algorithms. Let's take the general CART decision tree algorithm. Without going into much depth regarding information gain and impurity measures, we can think of the decision as \"is feature x_i >= some_val?\" Intuitively, we can see that it really doesn't matter on which scale this feature is (centimeters, Fahrenheit, a standardized scale -- it really doesn't matter). Some examples of algorithms where feature scaling matters are: k-nearest neighbors with an Euclidean distance measure if want all features to contribute equally k-means (see k-nearest neighbors) logistic regression, SVMs, perceptrons, neural networks etc. if you are using gradient descent/ascent-based optimization, otherwise some weights will update much faster than others linear discriminant analysis, principal component analysis, kernel principal component analysis since you want to find directions of maximizing the variance (under the constraints that those directions/eigenvectors/principal components are orthogonal); you want to have features on the same scale since you'd emphasize variables on \"larger measurement scales\" more. There are many more cases than I can possibly list here ... I always recommend you to think about the algorithm and what it's doing, and then it typically becomes obvious whether we want to scale your features or not. In addition, we'd also want to think about whether we want to \"standardize\" or \"normalize\" (here: scaling to [0, 1] range) our data. Some algorithms assume that our data is centered at 0. For example, if we initialize the weights of a small multi-layer perceptron with tanh activation units to 0 or small random values centered around zero, we want to update the model weights \"equally.\" As a rule of thumb I'd say: When in doubt, just standardize the data, it shouldn't hurt.","title":"MinMaxScaling: Min-max scaling fpr pandas DataFrames and NumPy arrays"},{"location":"user_guide/preprocessing/minmax_scaling/#example-1-scaling-a-pandas-dataframe","text":"import pandas as pd s1 = pd.Series([1, 2, 3, 4, 5, 6], index=(range(6))) s2 = pd.Series([10, 9, 8, 7, 6, 5], index=(range(6))) df = pd.DataFrame(s1, columns=['s1']) df['s2'] = s2 df .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } s1 s2 0 1 10 1 2 9 2 3 8 3 4 7 4 5 6 5 6 5 from mlxtend.preprocessing import minmax_scaling minmax_scaling(df, columns=['s1', 's2']) .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } s1 s2 0 0.0 1.0 1 0.2 0.8 2 0.4 0.6 3 0.6 0.4 4 0.8 0.2 5 1.0 0.0","title":"Example 1 - Scaling a Pandas DataFrame"},{"location":"user_guide/preprocessing/minmax_scaling/#example-2-scaling-a-numpy-array","text":"import numpy as np X = np.array([[1, 10], [2, 9], [3, 8], [4, 7], [5, 6], [6, 5]]) X array([[ 1, 10], [ 2, 9], [ 3, 8], [ 4, 7], [ 5, 6], [ 6, 5]]) from mlxtend.preprocessing import minmax_scaling minmax_scaling(X, columns=[0, 1]) array([[0. , 1. ], [0.2, 0.8], [0.4, 0.6], [0.6, 0.4], [0.8, 0.2], [1. , 0. ]])","title":"Example 2 - Scaling a NumPy Array"},{"location":"user_guide/preprocessing/minmax_scaling/#api","text":"minmax_scaling(array, columns, min_val=0, max_val=1) Min max scaling of pandas' DataFrames. Parameters array : pandas DataFrame or NumPy ndarray, shape = [n_rows, n_columns]. columns : array-like, shape = [n_columns] Array-like with column names, e.g., ['col1', 'col2', ...] or column indices [0, 2, 4, ...] min_val : int or float , optional (default= 0 ) minimum value after rescaling. max_val : int or float , optional (default= 1 ) maximum value after rescaling. Returns df_new : pandas DataFrame object. Copy of the array or DataFrame with rescaled columns. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/preprocessing/minmax_scaling/","title":"API"},{"location":"user_guide/preprocessing/one-hot_encoding/","text":"one_hot: One-Hot encoding function for class label arrays A function that performs one-hot encoding for class labels. from mlxtend.preprocessing import one_hot Overview Typical supervised machine learning algorithms for classifications assume that the class labels are nominal (a special case of categorical where no order is implied). A typical example of an nominal feature would be \"color\" since we can't say (in most applications) that \"orange > blue > red\". The one_hot function provides a simple interface to convert class label integers into a so-called one-hot array, where each unique label is represented as a column in the new array. For example, let's assume we have 5 data points from 3 different classes: 0, 1, and 2. y = [0, # sample 1, class 0 1, # sample 2, class 1 0, # sample 3, class 0 2, # sample 4, class 2 2] # sample 5, class 2 After one-hot encoding, we then obtain the following array (note that the index position of the \"1\" in each row denotes the class label of this sample): y = [[1, 0, 0], # sample 1, class 0 [0, 1, 0], # sample 2, class 1 [1, 0, 0], # sample 3, class 0 [0, 0, 1], # sample 4, class 2 [0, 0, 1] # sample 5, class 2 ]) Example 1 - Defaults from mlxtend.preprocessing import one_hot import numpy as np y = np.array([0, 1, 2, 1, 2]) one_hot(y) array([[ 1., 0., 0.], [ 0., 1., 0.], [ 0., 0., 1.], [ 0., 1., 0.], [ 0., 0., 1.]]) Example 2 - Python Lists from mlxtend.preprocessing import one_hot y = [0, 1, 2, 1, 2] one_hot(y) array([[ 1., 0., 0.], [ 0., 1., 0.], [ 0., 0., 1.], [ 0., 1., 0.], [ 0., 0., 1.]]) Example 3 - Integer Arrays from mlxtend.preprocessing import one_hot y = [0, 1, 2, 1, 2] one_hot(y, dtype='int') array([[1, 0, 0], [0, 1, 0], [0, 0, 1], [0, 1, 0], [0, 0, 1]]) Example 4 - Arbitrary Numbers of Class Labels from mlxtend.preprocessing import one_hot y = [0, 1, 2, 1, 2] one_hot(y, num_labels=10) array([[ 1., 0., 0., 0., 0., 0., 0., 0., 0., 0.], [ 0., 1., 0., 0., 0., 0., 0., 0., 0., 0.], [ 0., 0., 1., 0., 0., 0., 0., 0., 0., 0.], [ 0., 1., 0., 0., 0., 0., 0., 0., 0., 0.], [ 0., 0., 1., 0., 0., 0., 0., 0., 0., 0.]]) API one_hot(y, num_labels='auto', dtype='float') One-hot encoding of class labels Parameters y : array-like, shape = [n_classlabels] Python list or numpy array consisting of class labels. num_labels : int or 'auto' Number of unique labels in the class label array. Infers the number of unique labels from the input array if set to 'auto'. dtype : str NumPy array type (float, float32, float64) of the output array. Returns ary : numpy.ndarray, shape = [n_classlabels] One-hot encoded array, where each sample is represented as a row vector in the returned array. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/preprocessing/one_hot/","title":"One hot encoding"},{"location":"user_guide/preprocessing/one-hot_encoding/#one_hot-one-hot-encoding-function-for-class-label-arrays","text":"A function that performs one-hot encoding for class labels. from mlxtend.preprocessing import one_hot","title":"one_hot: One-Hot encoding function for class label arrays"},{"location":"user_guide/preprocessing/one-hot_encoding/#overview","text":"Typical supervised machine learning algorithms for classifications assume that the class labels are nominal (a special case of categorical where no order is implied). A typical example of an nominal feature would be \"color\" since we can't say (in most applications) that \"orange > blue > red\". The one_hot function provides a simple interface to convert class label integers into a so-called one-hot array, where each unique label is represented as a column in the new array. For example, let's assume we have 5 data points from 3 different classes: 0, 1, and 2. y = [0, # sample 1, class 0 1, # sample 2, class 1 0, # sample 3, class 0 2, # sample 4, class 2 2] # sample 5, class 2 After one-hot encoding, we then obtain the following array (note that the index position of the \"1\" in each row denotes the class label of this sample): y = [[1, 0, 0], # sample 1, class 0 [0, 1, 0], # sample 2, class 1 [1, 0, 0], # sample 3, class 0 [0, 0, 1], # sample 4, class 2 [0, 0, 1] # sample 5, class 2 ])","title":"Overview"},{"location":"user_guide/preprocessing/one-hot_encoding/#example-1-defaults","text":"from mlxtend.preprocessing import one_hot import numpy as np y = np.array([0, 1, 2, 1, 2]) one_hot(y) array([[ 1., 0., 0.], [ 0., 1., 0.], [ 0., 0., 1.], [ 0., 1., 0.], [ 0., 0., 1.]])","title":"Example 1 - Defaults"},{"location":"user_guide/preprocessing/one-hot_encoding/#example-2-python-lists","text":"from mlxtend.preprocessing import one_hot y = [0, 1, 2, 1, 2] one_hot(y) array([[ 1., 0., 0.], [ 0., 1., 0.], [ 0., 0., 1.], [ 0., 1., 0.], [ 0., 0., 1.]])","title":"Example 2 - Python Lists"},{"location":"user_guide/preprocessing/one-hot_encoding/#example-3-integer-arrays","text":"from mlxtend.preprocessing import one_hot y = [0, 1, 2, 1, 2] one_hot(y, dtype='int') array([[1, 0, 0], [0, 1, 0], [0, 0, 1], [0, 1, 0], [0, 0, 1]])","title":"Example 3 - Integer Arrays"},{"location":"user_guide/preprocessing/one-hot_encoding/#example-4-arbitrary-numbers-of-class-labels","text":"from mlxtend.preprocessing import one_hot y = [0, 1, 2, 1, 2] one_hot(y, num_labels=10) array([[ 1., 0., 0., 0., 0., 0., 0., 0., 0., 0.], [ 0., 1., 0., 0., 0., 0., 0., 0., 0., 0.], [ 0., 0., 1., 0., 0., 0., 0., 0., 0., 0.], [ 0., 1., 0., 0., 0., 0., 0., 0., 0., 0.], [ 0., 0., 1., 0., 0., 0., 0., 0., 0., 0.]])","title":"Example 4 - Arbitrary Numbers of Class Labels"},{"location":"user_guide/preprocessing/one-hot_encoding/#api","text":"one_hot(y, num_labels='auto', dtype='float') One-hot encoding of class labels Parameters y : array-like, shape = [n_classlabels] Python list or numpy array consisting of class labels. num_labels : int or 'auto' Number of unique labels in the class label array. Infers the number of unique labels from the input array if set to 'auto'. dtype : str NumPy array type (float, float32, float64) of the output array. Returns ary : numpy.ndarray, shape = [n_classlabels] One-hot encoded array, where each sample is represented as a row vector in the returned array. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/preprocessing/one_hot/","title":"API"},{"location":"user_guide/preprocessing/shuffle_arrays_unison/","text":"shuffle_arrays_unison: shuffle arrays in a consistent fashion A function for NumPy arrays in unison. from mlxtend.preprocessing import shuffle_arrays_unison Example 1 - Scaling a Pandas DataFrame import numpy as np from mlxtend.preprocessing import shuffle_arrays_unison X = np.array([[1, 2, 3], [4, 5, 6], [7, 8, 9]]) y = np.array([1, 2, 3]) print('X:\\n%s' % X) print('y:\\n%s' % y) X: [[1 2 3] [4 5 6] [7 8 9]] y: [1 2 3] X2, y2 = shuffle_arrays_unison(arrays=[X, y], random_seed=3) print('X2:\\n%s' % X2) print('y2:\\n%s' % y2) X2: [[4 5 6] [1 2 3] [7 8 9]] y2: [2 1 3] API shuffle_arrays_unison(arrays, random_seed=None) Shuffle NumPy arrays in unison. Parameters arrays : array-like, shape = [n_arrays] A list of NumPy arrays. random_seed : int (default: None) Sets the random state. Returns shuffled_arrays : A list of NumPy arrays after shuffling. Examples >>> import numpy as np >>> from mlxtend.preprocessing import shuffle_arrays_unison >>> X1 = np.array([[1, 2, 3], [4, 5, 6], [7, 8, 9]]) >>> y1 = np.array([1, 2, 3]) >>> X2, y2 = shuffle_arrays_unison(arrays=[X1, y1], random_seed=3) >>> assert(X2.all() == np.array([[4, 5, 6], [1, 2, 3], [7, 8, 9]]).all()) >>> assert(y2.all() == np.array([2, 1, 3]).all()) >>> For more usage examples, please see https://rasbt.github.io/mlxtend/user_guide/preprocessing/shuffle_arrays_unison/","title":"shuffle_arrays_unison: shuffle arrays in a consistent fashion"},{"location":"user_guide/preprocessing/shuffle_arrays_unison/#shuffle_arrays_unison-shuffle-arrays-in-a-consistent-fashion","text":"A function for NumPy arrays in unison. from mlxtend.preprocessing import shuffle_arrays_unison","title":"shuffle_arrays_unison: shuffle arrays in a consistent fashion"},{"location":"user_guide/preprocessing/shuffle_arrays_unison/#example-1-scaling-a-pandas-dataframe","text":"import numpy as np from mlxtend.preprocessing import shuffle_arrays_unison X = np.array([[1, 2, 3], [4, 5, 6], [7, 8, 9]]) y = np.array([1, 2, 3]) print('X:\\n%s' % X) print('y:\\n%s' % y) X: [[1 2 3] [4 5 6] [7 8 9]] y: [1 2 3] X2, y2 = shuffle_arrays_unison(arrays=[X, y], random_seed=3) print('X2:\\n%s' % X2) print('y2:\\n%s' % y2) X2: [[4 5 6] [1 2 3] [7 8 9]] y2: [2 1 3]","title":"Example 1 - Scaling a Pandas DataFrame"},{"location":"user_guide/preprocessing/shuffle_arrays_unison/#api","text":"shuffle_arrays_unison(arrays, random_seed=None) Shuffle NumPy arrays in unison. Parameters arrays : array-like, shape = [n_arrays] A list of NumPy arrays. random_seed : int (default: None) Sets the random state. Returns shuffled_arrays : A list of NumPy arrays after shuffling. Examples >>> import numpy as np >>> from mlxtend.preprocessing import shuffle_arrays_unison >>> X1 = np.array([[1, 2, 3], [4, 5, 6], [7, 8, 9]]) >>> y1 = np.array([1, 2, 3]) >>> X2, y2 = shuffle_arrays_unison(arrays=[X1, y1], random_seed=3) >>> assert(X2.all() == np.array([[4, 5, 6], [1, 2, 3], [7, 8, 9]]).all()) >>> assert(y2.all() == np.array([2, 1, 3]).all()) >>> For more usage examples, please see https://rasbt.github.io/mlxtend/user_guide/preprocessing/shuffle_arrays_unison/","title":"API"},{"location":"user_guide/preprocessing/standardize/","text":"standardize: A function to standardize columns in a 2D NumPy array A function that performs column-based standardization on a NumPy array. from mlxtend.preprocessing import standardize Overview The result of standardization (or Z-score normalization) is that the features will be rescaled so that they'll have the properties of a standard normal distribution with \\mu = 0 and \\sigma = 1 . where \\mu is the mean (average) and \\sigma is the standard deviation from the mean; standard scores (also called z scores) of the samples are calculated as z=\\frac{x-\\mu}{\\sigma}. Standardizing the features so that they are centered around 0 with a standard deviation of 1 is not only important if we are comparing measurements that have different units, but it is also a general requirement for the optimal performance of many machine learning algorithms. One family of algorithms that is scale-invariant encompasses tree-based learning algorithms. Let's take the general CART decision tree algorithm. Without going into much depth regarding information gain and impurity measures, we can think of the decision as \"is feature x_i >= some_val?\" Intuitively, we can see that it really doesn't matter on which scale this feature is (centimeters, Fahrenheit, a standardized scale -- it really doesn't matter). Some examples of algorithms where feature scaling matters are: k-nearest neighbors with an Euclidean distance measure if want all features to contribute equally k-means (see k-nearest neighbors) logistic regression, SVMs, perceptrons, neural networks etc. if you are using gradient descent/ascent-based optimization, otherwise some weights will update much faster than others linear discriminant analysis, principal component analysis, kernel principal component analysis since you want to find directions of maximizing the variance (under the constraints that those directions/eigenvectors/principal components are orthogonal); you want to have features on the same scale since you'd emphasize variables on \"larger measurement scales\" more. There are many more cases than I can possibly list here ... I always recommend you to think about the algorithm and what it's doing, and then it typically becomes obvious whether we want to scale your features or not. In addition, we'd also want to think about whether we want to \"standardize\" or \"normalize\" (here: scaling to [0, 1] range) our data. Some algorithms assume that our data is centered at 0. For example, if we initialize the weights of a small multi-layer perceptron with tanh activation units to 0 or small random values centered around zero, we want to update the model weights \"equally.\" As a rule of thumb I'd say: When in doubt, just standardize the data, it shouldn't hurt. Example 1 - Standardize a Pandas DataFrame import pandas as pd s1 = pd.Series([1, 2, 3, 4, 5, 6], index=(range(6))) s2 = pd.Series([10, 9, 8, 7, 6, 5], index=(range(6))) df = pd.DataFrame(s1, columns=['s1']) df['s2'] = s2 df s1 s2 0 1 10 1 2 9 2 3 8 3 4 7 4 5 6 5 6 5 from mlxtend.preprocessing import standardize standardize(df, columns=['s1', 's2']) s1 s2 0 -1.46385 1.46385 1 -0.87831 0.87831 2 -0.29277 0.29277 3 0.29277 -0.29277 4 0.87831 -0.87831 5 1.46385 -1.46385 Example 2 - Standardize a NumPy Array import numpy as np X = np.array([[1, 10], [2, 9], [3, 8], [4, 7], [5, 6], [6, 5]]) X array([[ 1, 10], [ 2, 9], [ 3, 8], [ 4, 7], [ 5, 6], [ 6, 5]]) from mlxtend.preprocessing import standardize standardize(X, columns=[0, 1]) array([[-1.46385011, 1.46385011], [-0.87831007, 0.87831007], [-0.29277002, 0.29277002], [ 0.29277002, -0.29277002], [ 0.87831007, -0.87831007], [ 1.46385011, -1.46385011]]) Example 3 - Re-using parameters In machine learning contexts, it is desired to re-use the parameters that have been obtained from a training set to scale new, future data (including the independent test set). By setting return_params=True , the standardize function returns a second object, a parameter dictionary containing the column means and standard deviations that can be re-used by feeding it to the params parameter upon function call. import numpy as np from mlxtend.preprocessing import standardize X_train = np.array([[1, 10], [4, 7], [3, 8]]) X_test = np.array([[1, 2], [3, 4], [5, 6]]) X_train_std, params = standardize(X_train, columns=[0, 1], return_params=True) X_train_std array([[-1.33630621, 1.33630621], [ 1.06904497, -1.06904497], [ 0.26726124, -0.26726124]]) params {'avgs': array([ 2.66666667, 8.33333333]), 'stds': array([ 1.24721913, 1.24721913])} X_test_std = standardize(X_test, columns=[0, 1], params=params) X_test_std array([[-1.33630621, -5.0779636 ], [ 0.26726124, -3.47439614], [ 1.87082869, -1.87082869]]) API standardize(array, columns=None, ddof=0, return_params=False, params=None) Standardize columns in pandas DataFrames. Parameters array : pandas DataFrame or NumPy ndarray, shape = [n_rows, n_columns]. columns : array-like, shape = [n_columns] (default: None) Array-like with column names, e.g., ['col1', 'col2', ...] or column indices [0, 2, 4, ...] If None, standardizes all columns. ddof : int (default: 0) Delta Degrees of Freedom. The divisor used in calculations is N - ddof, where N represents the number of elements. return_params : dict (default: False) If set to True, a dictionary is returned in addition to the standardized array. The parameter dictionary contains the column means ('avgs') and standard deviations ('stds') of the individual columns. params : dict (default: None) A dictionary with column means and standard deviations as returned by the standardize function if return_params was set to True. If a params dictionary is provided, the standardize function will use these instead of computing them from the current array. Notes If all values in a given column are the same, these values are all set to 0.0 . The standard deviation in the parameters dictionary is consequently set to 1.0 to avoid dividing by zero. Returns df_new : pandas DataFrame object. Copy of the array or DataFrame with standardized columns. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/preprocessing/standardize/","title":"standardize: A function to standardize columns in a 2D NumPy array"},{"location":"user_guide/preprocessing/standardize/#standardize-a-function-to-standardize-columns-in-a-2d-numpy-array","text":"A function that performs column-based standardization on a NumPy array. from mlxtend.preprocessing import standardize","title":"standardize: A function to standardize columns in a 2D NumPy array"},{"location":"user_guide/preprocessing/standardize/#overview","text":"The result of standardization (or Z-score normalization) is that the features will be rescaled so that they'll have the properties of a standard normal distribution with \\mu = 0 and \\sigma = 1 . where \\mu is the mean (average) and \\sigma is the standard deviation from the mean; standard scores (also called z scores) of the samples are calculated as z=\\frac{x-\\mu}{\\sigma}. Standardizing the features so that they are centered around 0 with a standard deviation of 1 is not only important if we are comparing measurements that have different units, but it is also a general requirement for the optimal performance of many machine learning algorithms. One family of algorithms that is scale-invariant encompasses tree-based learning algorithms. Let's take the general CART decision tree algorithm. Without going into much depth regarding information gain and impurity measures, we can think of the decision as \"is feature x_i >= some_val?\" Intuitively, we can see that it really doesn't matter on which scale this feature is (centimeters, Fahrenheit, a standardized scale -- it really doesn't matter). Some examples of algorithms where feature scaling matters are: k-nearest neighbors with an Euclidean distance measure if want all features to contribute equally k-means (see k-nearest neighbors) logistic regression, SVMs, perceptrons, neural networks etc. if you are using gradient descent/ascent-based optimization, otherwise some weights will update much faster than others linear discriminant analysis, principal component analysis, kernel principal component analysis since you want to find directions of maximizing the variance (under the constraints that those directions/eigenvectors/principal components are orthogonal); you want to have features on the same scale since you'd emphasize variables on \"larger measurement scales\" more. There are many more cases than I can possibly list here ... I always recommend you to think about the algorithm and what it's doing, and then it typically becomes obvious whether we want to scale your features or not. In addition, we'd also want to think about whether we want to \"standardize\" or \"normalize\" (here: scaling to [0, 1] range) our data. Some algorithms assume that our data is centered at 0. For example, if we initialize the weights of a small multi-layer perceptron with tanh activation units to 0 or small random values centered around zero, we want to update the model weights \"equally.\" As a rule of thumb I'd say: When in doubt, just standardize the data, it shouldn't hurt.","title":"Overview"},{"location":"user_guide/preprocessing/standardize/#example-1-standardize-a-pandas-dataframe","text":"import pandas as pd s1 = pd.Series([1, 2, 3, 4, 5, 6], index=(range(6))) s2 = pd.Series([10, 9, 8, 7, 6, 5], index=(range(6))) df = pd.DataFrame(s1, columns=['s1']) df['s2'] = s2 df s1 s2 0 1 10 1 2 9 2 3 8 3 4 7 4 5 6 5 6 5 from mlxtend.preprocessing import standardize standardize(df, columns=['s1', 's2']) s1 s2 0 -1.46385 1.46385 1 -0.87831 0.87831 2 -0.29277 0.29277 3 0.29277 -0.29277 4 0.87831 -0.87831 5 1.46385 -1.46385","title":"Example 1 - Standardize a Pandas DataFrame"},{"location":"user_guide/preprocessing/standardize/#example-2-standardize-a-numpy-array","text":"import numpy as np X = np.array([[1, 10], [2, 9], [3, 8], [4, 7], [5, 6], [6, 5]]) X array([[ 1, 10], [ 2, 9], [ 3, 8], [ 4, 7], [ 5, 6], [ 6, 5]]) from mlxtend.preprocessing import standardize standardize(X, columns=[0, 1]) array([[-1.46385011, 1.46385011], [-0.87831007, 0.87831007], [-0.29277002, 0.29277002], [ 0.29277002, -0.29277002], [ 0.87831007, -0.87831007], [ 1.46385011, -1.46385011]])","title":"Example 2 - Standardize a NumPy Array"},{"location":"user_guide/preprocessing/standardize/#example-3-re-using-parameters","text":"In machine learning contexts, it is desired to re-use the parameters that have been obtained from a training set to scale new, future data (including the independent test set). By setting return_params=True , the standardize function returns a second object, a parameter dictionary containing the column means and standard deviations that can be re-used by feeding it to the params parameter upon function call. import numpy as np from mlxtend.preprocessing import standardize X_train = np.array([[1, 10], [4, 7], [3, 8]]) X_test = np.array([[1, 2], [3, 4], [5, 6]]) X_train_std, params = standardize(X_train, columns=[0, 1], return_params=True) X_train_std array([[-1.33630621, 1.33630621], [ 1.06904497, -1.06904497], [ 0.26726124, -0.26726124]]) params {'avgs': array([ 2.66666667, 8.33333333]), 'stds': array([ 1.24721913, 1.24721913])} X_test_std = standardize(X_test, columns=[0, 1], params=params) X_test_std array([[-1.33630621, -5.0779636 ], [ 0.26726124, -3.47439614], [ 1.87082869, -1.87082869]])","title":"Example 3 - Re-using parameters"},{"location":"user_guide/preprocessing/standardize/#api","text":"standardize(array, columns=None, ddof=0, return_params=False, params=None) Standardize columns in pandas DataFrames. Parameters array : pandas DataFrame or NumPy ndarray, shape = [n_rows, n_columns]. columns : array-like, shape = [n_columns] (default: None) Array-like with column names, e.g., ['col1', 'col2', ...] or column indices [0, 2, 4, ...] If None, standardizes all columns. ddof : int (default: 0) Delta Degrees of Freedom. The divisor used in calculations is N - ddof, where N represents the number of elements. return_params : dict (default: False) If set to True, a dictionary is returned in addition to the standardized array. The parameter dictionary contains the column means ('avgs') and standard deviations ('stds') of the individual columns. params : dict (default: None) A dictionary with column means and standard deviations as returned by the standardize function if return_params was set to True. If a params dictionary is provided, the standardize function will use these instead of computing them from the current array. Notes If all values in a given column are the same, these values are all set to 0.0 . The standard deviation in the parameters dictionary is consequently set to 1.0 to avoid dividing by zero. Returns df_new : pandas DataFrame object. Copy of the array or DataFrame with standardized columns. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/preprocessing/standardize/","title":"API"},{"location":"user_guide/regressor/LinearRegression/","text":"LinearRegression: An implementation of ordinary least-squares linear regression A implementation of Ordinary Least Squares simple and multiple linear regression. from mlxtend.regressor import LinearRegression Overview Illustration of a simple linear regression model: In Ordinary Least Squares (OLS) Linear Regression, our goal is to find the line (or hyperplane) that minimizes the vertical offsets. Or in other words, we define the best-fitting line as the line that minimizes the sum of squared errors (SSE) or mean squared error (MSE) between our target variable (y) and our predicted output over all samples i in our dataset of size n . SSE = \\sum_i \\big(\\text{target}^{(i)} - \\text{output}^{(i)}\\big)^2 MSE = \\frac{1}{n} \\times SSE Now, LinearRegression implements a linear regression model for performing ordinary least squares regression using one of the following five approaches: Normal Equations QR Decomposition Method SVD (Singular Value Decomposition) method Gradient Descent Stochastic Gradient Descent Normal Equations (closed-form solution) The closed-form solution should be preferred for \"smaller\" datasets where calculating (a \"costly\") matrix inverse is not a concern. For very large datasets, or datasets where the inverse of [X^T X] may not exist (the matrix is non-invertible or singular, e.g., in case of perfect multicollinearity), the QR, SVD or gradient descent approaches are to be preferred. The linear function (linear regression model) is defined as: y = w_0x_0 + w_1x_1 + ... + w_mx_m = \\sum_{i=0}^{m} = \\mathbf{w}^T\\mathbf{x} where y is the response variable, \\mathbf{x} is an m -dimensional sample vector, and \\mathbf{w} is the weight vector (vector of coefficients). Note that w_0 represents the y-axis intercept of the model and therefore x_0=1 . Using the closed-form solution (normal equation), we compute the weights of the model as follows: \\mathbf{w} = (\\mathbf{X}^T\\mathbf{X})^{-1}\\mathbf{X}^Ty Stable OLS via QR Factorization The QR decomposition method offers a more numerically stable alternative to the closed-form, analytical solution based on the \"normal equations,\" and it can be used to compute the inverse of large matrices more efficiently. QR decomposition method decomposes given matrix into two matrices for which an inverse can be easily obtained. For instance, a given matrix X \\in \\mathbb{R}^{n \\times m} , the QR decomposition into two matrices is: \\mathbf{X = QR}, where Q \\in \\mathbb{R}^{n \\times m} is an orthonormal matrix, such that Q^\\top Q = QQ^\\top = I . The second matrix R \\in \\mathbf{R}^{m \\times m} is an upper triangular matrix. The weight parameters of the ordinary least squares regression model can then be computed as follows [1]: \\mathbf{w} = \\mathbf{R}^{-1}\\mathbf{Q}^\\top y. Stable OLS via Singular Value Decomposition Another alternative way for obtaining the OLS model weights in a numerically stable fashion is by Singular Value Decomposition (SVD), which is defined as: \\mathbf{X}=\\mathbf{U}\\Sigma \\mathbf{V}^\\top, for a given matrix \\mathbf{X} . Then, it can be shown that the pseudo-inverse of X , X^+ , can be obtained as follows [1]: X^+ = \\mathbf{U} \\mathbf{\\Sigma}^+ V^{\\top}. Note that while \\Sigma is the diagonal matrix consisting of singular values of \\mathbf{X} , \\Sigma^{+} is the diagonal matrix consisting of the reciprocals of the singular values. The model weights can then be computed as follows: \\mathbf{w} = \\mathbf{X}^+ \\mathbf{y}. Please note that this OLS method is computationally most inefficient. However, it is a useful approach when the direct method (normal equations) or QR factorization cannot be applied or the normal equations (via \\mathbf{X}^T \\mathbf{X} ) are ill-conditioned [3]. Gradient Descent (GD) and Stochastic Gradient Descent (SGD) See Gradient Descent and Stochastic Gradient Descent and Deriving the Gradient Descent Rule for Linear Regression and Adaline for details. Random shuffling is implemented as: for one or more epochs randomly shuffle samples in the training set for training sample i compute gradients and perform weight updates References [1] Chapter 3, page 55, Linear Methods for Regression. Trevor Hastie; Robert Tibshirani; Jerome Friedman (2009). The Elements of Statistical Learning: Data Mining, Inference, and Prediction (2nd ed.) . New York: Springer. (ISBN 978\u20130\u2013387\u201384858\u20137) [2] G. Strang, Linear Algebra and Its Applications, 2nd Ed., Orlando, FL, Academic Press, Inc., 1980, pp. 139-142. [3] Douglas Wilhelm Harder. Numerical Analysis for Engineering. Section 4.8, ill-conditioned Matrices Example 1 - Closed Form Solution import numpy as np import matplotlib.pyplot as plt from mlxtend.regressor import LinearRegression X = np.array([ 1.0, 2.1, 3.6, 4.2, 6])[:, np.newaxis] y = np.array([ 1.0, 2.0, 3.0, 4.0, 5.0]) ne_lr = LinearRegression() ne_lr.fit(X, y) print('Intercept: %.2f' % ne_lr.b_) print('Slope: %.2f' % ne_lr.w_[0]) def lin_regplot(X, y, model): plt.scatter(X, y, c='blue') plt.plot(X, model.predict(X), color='red') return lin_regplot(X, y, ne_lr) plt.show() Intercept: 0.25 Slope: 0.81 Example 2 - QR decomposition method import numpy as np import matplotlib.pyplot as plt from mlxtend.regressor import LinearRegression X = np.array([ 1.0, 2.1, 3.6, 4.2, 6])[:, np.newaxis] y = np.array([ 1.0, 2.0, 3.0, 4.0, 5.0]) qr_lr = LinearRegression(method='qr') qr_lr.fit(X, y) print('Intercept: %.2f' % qr_lr.b_) print('Slope: %.2f' % qr_lr.w_[0]) def lin_regplot(X, y, model): plt.scatter(X, y, c='blue') plt.plot(X, model.predict(X), color='red') return lin_regplot(X, y, qr_lr) plt.show() Intercept: 0.25 Slope: 0.81 Example 3 - SVD method import numpy as np import matplotlib.pyplot as plt from mlxtend.regressor import LinearRegression X = np.array([ 1.0, 2.1, 3.6, 4.2, 6])[:, np.newaxis] y = np.array([ 1.0, 2.0, 3.0, 4.0, 5.0]) svd_lr = LinearRegression(method='svd') svd_lr.fit(X, y) print('Intercept: %.2f' %svd_lr.b_) print('Slope: %.2f' % svd_lr.w_[0]) def lin_regplot(X, y, model): plt.scatter(X, y, c='blue') plt.plot(X, model.predict(X), color='red') return lin_regplot(X, y, svd_lr) plt.show() Intercept: 0.25 Slope: 0.81 Example 4 - Gradient Descent import numpy as np import matplotlib.pyplot as plt from mlxtend.regressor import LinearRegression X = np.array([ 1.0, 2.1, 3.6, 4.2, 6])[:, np.newaxis] y = np.array([ 1.0, 2.0, 3.0, 4.0, 5.0]) gd_lr = LinearRegression(method='sgd', eta=0.005, epochs=100, minibatches=1, random_seed=123, print_progress=3) gd_lr.fit(X, y) print('Intercept: %.2f' % gd_lr.b_) print('Slope: %.2f' % gd_lr.w_) def lin_regplot(X, y, model): plt.scatter(X, y, c='blue') plt.plot(X, model.predict(X), color='red') return lin_regplot(X, y, gd_lr) plt.show() Iteration: 100/100 | Cost 0.08 | Elapsed: 0:00:00 | ETA: 0:00:00 Intercept: 0.22 Slope: 0.82 # Visualizing the cost to check for convergence and plotting the linear model: plt.plot(range(1, gd_lr.epochs+1), gd_lr.cost_) plt.xlabel('Epochs') plt.ylabel('Cost') plt.ylim([0, 0.2]) plt.tight_layout() plt.show() Example 5 - Stochastic Gradient Descent import numpy as np import matplotlib.pyplot as plt from mlxtend.regressor import LinearRegression X = np.array([ 1.0, 2.1, 3.6, 4.2, 6])[:, np.newaxis] y = np.array([ 1.0, 2.0, 3.0, 4.0, 5.0]) sgd_lr = LinearRegression(method='sgd', eta=0.01, epochs=100, random_seed=0, minibatches=len(y)) sgd_lr.fit(X, y) print('Intercept: %.2f' % sgd_lr.w_) print('Slope: %.2f' % sgd_lr.b_) def lin_regplot(X, y, model): plt.scatter(X, y, c='blue') plt.plot(X, model.predict(X), color='red') return lin_regplot(X, y, sgd_lr) plt.show() Intercept: 0.82 Slope: 0.24 plt.plot(range(1, sgd_lr.epochs+1), sgd_lr.cost_) plt.xlabel('Epochs') plt.ylabel('Cost') plt.ylim([0, 0.2]) plt.tight_layout() plt.show() Example 6 - Stochastic Gradient Descent with Minibatches import numpy as np import matplotlib.pyplot as plt from mlxtend.regressor import LinearRegression X = np.array([ 1.0, 2.1, 3.6, 4.2, 6])[:, np.newaxis] y = np.array([ 1.0, 2.0, 3.0, 4.0, 5.0]) sgd_lr = LinearRegression(method='sgd', eta=0.01, epochs=100, random_seed=0, minibatches=3) sgd_lr.fit(X, y) print('Intercept: %.2f' % sgd_lr.b_) print('Slope: %.2f' % sgd_lr.w_) def lin_regplot(X, y, model): plt.scatter(X, y, c='blue') plt.plot(X, model.predict(X), color='red') return lin_regplot(X, y, sgd_lr) plt.show() Intercept: 0.24 Slope: 0.82 plt.plot(range(1, sgd_lr.epochs+1), sgd_lr.cost_) plt.xlabel('Epochs') plt.ylabel('Cost') plt.ylim([0, 0.2]) plt.tight_layout() plt.show() API LinearRegression(method='direct', eta=0.01, epochs=50, minibatches=None, random_seed=None, print_progress=0) Ordinary least squares linear regression. Parameters method : string (default: 'direct') For gradient descent-based optimization, use sgd (see minibatch parameter for further options). Otherwise, if direct (default), the analytical method is used. For alternative, numerically more stable solutions, use either qr (QR decomopisition) or svd (Singular Value Decomposition). eta : float (default: 0.01) solver learning rate (between 0.0 and 1.0). Used with method = 'sgd' . (See methods parameter for details) epochs : int (default: 50) Passes over the training dataset. Prior to each epoch, the dataset is shuffled if minibatches > 1 to prevent cycles in stochastic gradient descent. Used with method = 'sgd' . (See methods parameter for details) minibatches : int (default: None) The number of minibatches for gradient-based optimization. If None: Direct method, QR, or SVD method (see method parameter for details) If 1: Gradient Descent learning If len(y): Stochastic Gradient Descent learning If 1 < minibatches < len(y): Minibatch learning random_seed : int (default: None) Set random state for shuffling and initializing the weights. Used in method = 'sgd' . (See methods parameter for details) print_progress : int (default: 0) Prints progress in fitting to stderr if method = 'sgd' . 0: No output 1: Epochs elapsed and cost 2: 1 plus time elapsed 3: 2 plus estimated time until completion Attributes w_ : 2d-array, shape={n_features, 1} Model weights after fitting. b_ : 1d-array, shape={1,} Bias unit after fitting. cost_ : list Sum of squared errors after each epoch; ignored if solver='normal equation' Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/regressor/LinearRegression/ Methods fit(X, y, init_params=True) Learn model from training data. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values. init_params : bool (default: True) Re-initializes model parameters prior to fitting. Set False to continue training with weights from a previous model fitting. Returns self : object get_params(deep=True) Get parameters for this estimator. Parameters deep : boolean, optional If True, will return the parameters for this estimator and contained subobjects that are estimators. Returns params : mapping of string to any Parameter names mapped to their values.' adapted from https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/base.py Author: Gael Varoquaux gael.varoquaux@normalesup.org License: BSD 3 clause predict(X) Predict targets from X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns target_values : array-like, shape = [n_samples] Predicted target values. set_params( params) Set the parameters of this estimator. The method works on simple estimators as well as on nested objects (such as pipelines). The latter have parameters of the form <component>__<parameter> so that it's possible to update each component of a nested object. Returns self adapted from https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/base.py Author: Gael Varoquaux gael.varoquaux@normalesup.org License: BSD 3 clause ython","title":"LinearRegression: An implementation of ordinary least-squares linear regression"},{"location":"user_guide/regressor/LinearRegression/#linearregression-an-implementation-of-ordinary-least-squares-linear-regression","text":"A implementation of Ordinary Least Squares simple and multiple linear regression. from mlxtend.regressor import LinearRegression","title":"LinearRegression: An implementation of ordinary least-squares linear regression"},{"location":"user_guide/regressor/LinearRegression/#overview","text":"Illustration of a simple linear regression model: In Ordinary Least Squares (OLS) Linear Regression, our goal is to find the line (or hyperplane) that minimizes the vertical offsets. Or in other words, we define the best-fitting line as the line that minimizes the sum of squared errors (SSE) or mean squared error (MSE) between our target variable (y) and our predicted output over all samples i in our dataset of size n . SSE = \\sum_i \\big(\\text{target}^{(i)} - \\text{output}^{(i)}\\big)^2 MSE = \\frac{1}{n} \\times SSE Now, LinearRegression implements a linear regression model for performing ordinary least squares regression using one of the following five approaches: Normal Equations QR Decomposition Method SVD (Singular Value Decomposition) method Gradient Descent Stochastic Gradient Descent","title":"Overview"},{"location":"user_guide/regressor/LinearRegression/#normal-equations-closed-form-solution","text":"The closed-form solution should be preferred for \"smaller\" datasets where calculating (a \"costly\") matrix inverse is not a concern. For very large datasets, or datasets where the inverse of [X^T X] may not exist (the matrix is non-invertible or singular, e.g., in case of perfect multicollinearity), the QR, SVD or gradient descent approaches are to be preferred. The linear function (linear regression model) is defined as: y = w_0x_0 + w_1x_1 + ... + w_mx_m = \\sum_{i=0}^{m} = \\mathbf{w}^T\\mathbf{x} where y is the response variable, \\mathbf{x} is an m -dimensional sample vector, and \\mathbf{w} is the weight vector (vector of coefficients). Note that w_0 represents the y-axis intercept of the model and therefore x_0=1 . Using the closed-form solution (normal equation), we compute the weights of the model as follows: \\mathbf{w} = (\\mathbf{X}^T\\mathbf{X})^{-1}\\mathbf{X}^Ty","title":"Normal Equations (closed-form solution)"},{"location":"user_guide/regressor/LinearRegression/#stable-ols-via-qr-factorization","text":"The QR decomposition method offers a more numerically stable alternative to the closed-form, analytical solution based on the \"normal equations,\" and it can be used to compute the inverse of large matrices more efficiently. QR decomposition method decomposes given matrix into two matrices for which an inverse can be easily obtained. For instance, a given matrix X \\in \\mathbb{R}^{n \\times m} , the QR decomposition into two matrices is: \\mathbf{X = QR}, where Q \\in \\mathbb{R}^{n \\times m} is an orthonormal matrix, such that Q^\\top Q = QQ^\\top = I . The second matrix R \\in \\mathbf{R}^{m \\times m} is an upper triangular matrix. The weight parameters of the ordinary least squares regression model can then be computed as follows [1]: \\mathbf{w} = \\mathbf{R}^{-1}\\mathbf{Q}^\\top y.","title":"Stable OLS via QR Factorization"},{"location":"user_guide/regressor/LinearRegression/#stable-ols-via-singular-value-decomposition","text":"Another alternative way for obtaining the OLS model weights in a numerically stable fashion is by Singular Value Decomposition (SVD), which is defined as: \\mathbf{X}=\\mathbf{U}\\Sigma \\mathbf{V}^\\top, for a given matrix \\mathbf{X} . Then, it can be shown that the pseudo-inverse of X , X^+ , can be obtained as follows [1]: X^+ = \\mathbf{U} \\mathbf{\\Sigma}^+ V^{\\top}. Note that while \\Sigma is the diagonal matrix consisting of singular values of \\mathbf{X} , \\Sigma^{+} is the diagonal matrix consisting of the reciprocals of the singular values. The model weights can then be computed as follows: \\mathbf{w} = \\mathbf{X}^+ \\mathbf{y}. Please note that this OLS method is computationally most inefficient. However, it is a useful approach when the direct method (normal equations) or QR factorization cannot be applied or the normal equations (via \\mathbf{X}^T \\mathbf{X} ) are ill-conditioned [3].","title":"Stable OLS via Singular Value Decomposition"},{"location":"user_guide/regressor/LinearRegression/#gradient-descent-gd-and-stochastic-gradient-descent-sgd","text":"See Gradient Descent and Stochastic Gradient Descent and Deriving the Gradient Descent Rule for Linear Regression and Adaline for details. Random shuffling is implemented as: for one or more epochs randomly shuffle samples in the training set for training sample i compute gradients and perform weight updates","title":"Gradient Descent (GD)  and Stochastic Gradient Descent (SGD)"},{"location":"user_guide/regressor/LinearRegression/#references","text":"[1] Chapter 3, page 55, Linear Methods for Regression. Trevor Hastie; Robert Tibshirani; Jerome Friedman (2009). The Elements of Statistical Learning: Data Mining, Inference, and Prediction (2nd ed.) . New York: Springer. (ISBN 978\u20130\u2013387\u201384858\u20137) [2] G. Strang, Linear Algebra and Its Applications, 2nd Ed., Orlando, FL, Academic Press, Inc., 1980, pp. 139-142. [3] Douglas Wilhelm Harder. Numerical Analysis for Engineering. Section 4.8, ill-conditioned Matrices","title":"References"},{"location":"user_guide/regressor/LinearRegression/#example-1-closed-form-solution","text":"import numpy as np import matplotlib.pyplot as plt from mlxtend.regressor import LinearRegression X = np.array([ 1.0, 2.1, 3.6, 4.2, 6])[:, np.newaxis] y = np.array([ 1.0, 2.0, 3.0, 4.0, 5.0]) ne_lr = LinearRegression() ne_lr.fit(X, y) print('Intercept: %.2f' % ne_lr.b_) print('Slope: %.2f' % ne_lr.w_[0]) def lin_regplot(X, y, model): plt.scatter(X, y, c='blue') plt.plot(X, model.predict(X), color='red') return lin_regplot(X, y, ne_lr) plt.show() Intercept: 0.25 Slope: 0.81","title":"Example 1 - Closed Form Solution"},{"location":"user_guide/regressor/LinearRegression/#example-2-qr-decomposition-method","text":"import numpy as np import matplotlib.pyplot as plt from mlxtend.regressor import LinearRegression X = np.array([ 1.0, 2.1, 3.6, 4.2, 6])[:, np.newaxis] y = np.array([ 1.0, 2.0, 3.0, 4.0, 5.0]) qr_lr = LinearRegression(method='qr') qr_lr.fit(X, y) print('Intercept: %.2f' % qr_lr.b_) print('Slope: %.2f' % qr_lr.w_[0]) def lin_regplot(X, y, model): plt.scatter(X, y, c='blue') plt.plot(X, model.predict(X), color='red') return lin_regplot(X, y, qr_lr) plt.show() Intercept: 0.25 Slope: 0.81","title":"Example 2 - QR decomposition method"},{"location":"user_guide/regressor/LinearRegression/#example-3-svd-method","text":"import numpy as np import matplotlib.pyplot as plt from mlxtend.regressor import LinearRegression X = np.array([ 1.0, 2.1, 3.6, 4.2, 6])[:, np.newaxis] y = np.array([ 1.0, 2.0, 3.0, 4.0, 5.0]) svd_lr = LinearRegression(method='svd') svd_lr.fit(X, y) print('Intercept: %.2f' %svd_lr.b_) print('Slope: %.2f' % svd_lr.w_[0]) def lin_regplot(X, y, model): plt.scatter(X, y, c='blue') plt.plot(X, model.predict(X), color='red') return lin_regplot(X, y, svd_lr) plt.show() Intercept: 0.25 Slope: 0.81","title":"Example 3 - SVD method"},{"location":"user_guide/regressor/LinearRegression/#example-4-gradient-descent","text":"import numpy as np import matplotlib.pyplot as plt from mlxtend.regressor import LinearRegression X = np.array([ 1.0, 2.1, 3.6, 4.2, 6])[:, np.newaxis] y = np.array([ 1.0, 2.0, 3.0, 4.0, 5.0]) gd_lr = LinearRegression(method='sgd', eta=0.005, epochs=100, minibatches=1, random_seed=123, print_progress=3) gd_lr.fit(X, y) print('Intercept: %.2f' % gd_lr.b_) print('Slope: %.2f' % gd_lr.w_) def lin_regplot(X, y, model): plt.scatter(X, y, c='blue') plt.plot(X, model.predict(X), color='red') return lin_regplot(X, y, gd_lr) plt.show() Iteration: 100/100 | Cost 0.08 | Elapsed: 0:00:00 | ETA: 0:00:00 Intercept: 0.22 Slope: 0.82 # Visualizing the cost to check for convergence and plotting the linear model: plt.plot(range(1, gd_lr.epochs+1), gd_lr.cost_) plt.xlabel('Epochs') plt.ylabel('Cost') plt.ylim([0, 0.2]) plt.tight_layout() plt.show()","title":"Example 4 - Gradient Descent"},{"location":"user_guide/regressor/LinearRegression/#example-5-stochastic-gradient-descent","text":"import numpy as np import matplotlib.pyplot as plt from mlxtend.regressor import LinearRegression X = np.array([ 1.0, 2.1, 3.6, 4.2, 6])[:, np.newaxis] y = np.array([ 1.0, 2.0, 3.0, 4.0, 5.0]) sgd_lr = LinearRegression(method='sgd', eta=0.01, epochs=100, random_seed=0, minibatches=len(y)) sgd_lr.fit(X, y) print('Intercept: %.2f' % sgd_lr.w_) print('Slope: %.2f' % sgd_lr.b_) def lin_regplot(X, y, model): plt.scatter(X, y, c='blue') plt.plot(X, model.predict(X), color='red') return lin_regplot(X, y, sgd_lr) plt.show() Intercept: 0.82 Slope: 0.24 plt.plot(range(1, sgd_lr.epochs+1), sgd_lr.cost_) plt.xlabel('Epochs') plt.ylabel('Cost') plt.ylim([0, 0.2]) plt.tight_layout() plt.show()","title":"Example 5 - Stochastic Gradient Descent"},{"location":"user_guide/regressor/LinearRegression/#example-6-stochastic-gradient-descent-with-minibatches","text":"import numpy as np import matplotlib.pyplot as plt from mlxtend.regressor import LinearRegression X = np.array([ 1.0, 2.1, 3.6, 4.2, 6])[:, np.newaxis] y = np.array([ 1.0, 2.0, 3.0, 4.0, 5.0]) sgd_lr = LinearRegression(method='sgd', eta=0.01, epochs=100, random_seed=0, minibatches=3) sgd_lr.fit(X, y) print('Intercept: %.2f' % sgd_lr.b_) print('Slope: %.2f' % sgd_lr.w_) def lin_regplot(X, y, model): plt.scatter(X, y, c='blue') plt.plot(X, model.predict(X), color='red') return lin_regplot(X, y, sgd_lr) plt.show() Intercept: 0.24 Slope: 0.82 plt.plot(range(1, sgd_lr.epochs+1), sgd_lr.cost_) plt.xlabel('Epochs') plt.ylabel('Cost') plt.ylim([0, 0.2]) plt.tight_layout() plt.show()","title":"Example 6 - Stochastic Gradient Descent with Minibatches"},{"location":"user_guide/regressor/LinearRegression/#api","text":"LinearRegression(method='direct', eta=0.01, epochs=50, minibatches=None, random_seed=None, print_progress=0) Ordinary least squares linear regression. Parameters method : string (default: 'direct') For gradient descent-based optimization, use sgd (see minibatch parameter for further options). Otherwise, if direct (default), the analytical method is used. For alternative, numerically more stable solutions, use either qr (QR decomopisition) or svd (Singular Value Decomposition). eta : float (default: 0.01) solver learning rate (between 0.0 and 1.0). Used with method = 'sgd' . (See methods parameter for details) epochs : int (default: 50) Passes over the training dataset. Prior to each epoch, the dataset is shuffled if minibatches > 1 to prevent cycles in stochastic gradient descent. Used with method = 'sgd' . (See methods parameter for details) minibatches : int (default: None) The number of minibatches for gradient-based optimization. If None: Direct method, QR, or SVD method (see method parameter for details) If 1: Gradient Descent learning If len(y): Stochastic Gradient Descent learning If 1 < minibatches < len(y): Minibatch learning random_seed : int (default: None) Set random state for shuffling and initializing the weights. Used in method = 'sgd' . (See methods parameter for details) print_progress : int (default: 0) Prints progress in fitting to stderr if method = 'sgd' . 0: No output 1: Epochs elapsed and cost 2: 1 plus time elapsed 3: 2 plus estimated time until completion Attributes w_ : 2d-array, shape={n_features, 1} Model weights after fitting. b_ : 1d-array, shape={1,} Bias unit after fitting. cost_ : list Sum of squared errors after each epoch; ignored if solver='normal equation' Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/regressor/LinearRegression/","title":"API"},{"location":"user_guide/regressor/LinearRegression/#methods","text":"fit(X, y, init_params=True) Learn model from training data. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values. init_params : bool (default: True) Re-initializes model parameters prior to fitting. Set False to continue training with weights from a previous model fitting. Returns self : object get_params(deep=True) Get parameters for this estimator. Parameters deep : boolean, optional If True, will return the parameters for this estimator and contained subobjects that are estimators. Returns params : mapping of string to any Parameter names mapped to their values.' adapted from https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/base.py Author: Gael Varoquaux gael.varoquaux@normalesup.org License: BSD 3 clause predict(X) Predict targets from X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns target_values : array-like, shape = [n_samples] Predicted target values. set_params( params) Set the parameters of this estimator. The method works on simple estimators as well as on nested objects (such as pipelines). The latter have parameters of the form <component>__<parameter> so that it's possible to update each component of a nested object. Returns self adapted from https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/base.py Author: Gael Varoquaux gael.varoquaux@normalesup.org License: BSD 3 clause ython","title":"Methods"},{"location":"user_guide/regressor/StackingCVRegressor/","text":"StackingCVRegressor: stacking with cross-validation for regression An ensemble-learning meta-regressor for stacking regression from mlxtend.regressor import StackingCVRegressor Overview Stacking is an ensemble learning technique to combine multiple regression models via a meta-regressor. The StackingCVRegressor extends the standard stacking algorithm (implemented as StackingRegressor ) using out-of-fold predictions to prepare the input data for the level-2 regressor. In the standard stacking procedure, the first-level regressors are fit to the same training set that is used prepare the inputs for the second-level regressor, which may lead to overfitting. The StackingCVRegressor , however, uses the concept of out-of-fold predictions: the dataset is split into k folds, and in k successive rounds, k-1 folds are used to fit the first level regressor. In each round, the first-level regressors are then applied to the remaining 1 subset that was not used for model fitting in each iteration. The resulting predictions are then stacked and provided -- as input data -- to the second-level regressor. After the training of the StackingCVRegressor , the first-level regressors are fit to the entire dataset for optimal predicitons. References Breiman, Leo. \" Stacked regressions. \" Machine learning 24.1 (1996): 49-64. Analogous implementation: StackingCVClassifier Example 1: Boston Housing Data Predictions In this example we evaluate some basic prediction models on the boston housing dataset and see how the R^2 and MSE scores are affected by combining the models with StackingCVRegressor . The code output below demonstrates that the stacked model performs the best on this dataset -- slightly better than the best single regression model. from mlxtend.regressor import StackingCVRegressor from sklearn.datasets import load_boston from sklearn.svm import SVR from sklearn.linear_model import Lasso from sklearn.ensemble import RandomForestRegressor from sklearn.model_selection import cross_val_score import numpy as np RANDOM_SEED = 42 X, y = load_boston(return_X_y=True) svr = SVR(kernel='linear') lasso = Lasso() rf = RandomForestRegressor(n_estimators=5, random_state=RANDOM_SEED) # Starting from v0.16.0, StackingCVRegressor supports # `random_state` to get deterministic result. stack = StackingCVRegressor(regressors=(svr, lasso, rf), meta_regressor=lasso, random_state=RANDOM_SEED) print('5-fold cross validation scores:\\n') for clf, label in zip([svr, lasso, rf, stack], ['SVM', 'Lasso', 'Random Forest', 'StackingCVRegressor']): scores = cross_val_score(clf, X, y, cv=5) print(\"R^2 Score: %0.2f (+/- %0.2f) [%s]\" % ( scores.mean(), scores.std(), label)) 5-fold cross validation scores: R^2 Score: 0.46 (+/- 0.29) [SVM] R^2 Score: 0.43 (+/- 0.14) [Lasso] R^2 Score: 0.53 (+/- 0.28) [Random Forest] R^2 Score: 0.57 (+/- 0.24) [StackingCVRegressor] stack = StackingCVRegressor(regressors=(svr, lasso, rf), meta_regressor=lasso) print('5-fold cross validation scores:\\n') for clf, label in zip([svr, lasso, rf, stack], ['SVM', 'Lasso', 'Random Forest', 'StackingCVRegressor']): scores = cross_val_score(clf, X, y, cv=5, scoring='neg_mean_squared_error') print(\"Neg. MSE Score: %0.2f (+/- %0.2f) [%s]\" % ( scores.mean(), scores.std(), label)) 5-fold cross validation scores: Neg. MSE Score: -33.33 (+/- 22.36) [SVM] Neg. MSE Score: -35.53 (+/- 16.99) [Lasso] Neg. MSE Score: -27.08 (+/- 15.67) [Random Forest] Neg. MSE Score: -25.85 (+/- 17.22) [StackingCVRegressor] Example 2: GridSearchCV with Stacking In this second example we demonstrate how StackingCVRegressor works in combination with GridSearchCV . The stack still allows tuning hyper parameters of the base and meta models! For instance, we can use estimator.get_params().keys() to get a full list of tunable parameters. from mlxtend.regressor import StackingCVRegressor from sklearn.datasets import load_boston from sklearn.linear_model import Lasso from sklearn.linear_model import Ridge from sklearn.ensemble import RandomForestRegressor from sklearn.model_selection import GridSearchCV X, y = load_boston(return_X_y=True) ridge = Ridge(random_state=RANDOM_SEED) lasso = Lasso(random_state=RANDOM_SEED) rf = RandomForestRegressor(random_state=RANDOM_SEED) stack = StackingCVRegressor(regressors=(lasso, ridge), meta_regressor=rf, random_state=RANDOM_SEED, use_features_in_secondary=True) params = {'lasso__alpha': [0.1, 1.0, 10.0], 'ridge__alpha': [0.1, 1.0, 10.0]} grid = GridSearchCV( estimator=stack, param_grid={ 'lasso__alpha': [x/5.0 for x in range(1, 10)], 'ridge__alpha': [x/20.0 for x in range(1, 10)], 'meta_regressor__n_estimators': [10, 100] }, cv=5, refit=True ) grid.fit(X, y) print(\"Best: %f using %s\" % (grid.best_score_, grid.best_params_)) Best: 0.679151 using {'lasso__alpha': 1.6, 'meta_regressor__n_estimators': 10, 'ridge__alpha': 0.4} cv_keys = ('mean_test_score', 'std_test_score', 'params') for r, _ in enumerate(grid.cv_results_['mean_test_score']): print(\"%0.3f +/- %0.2f %r\" % (grid.cv_results_[cv_keys[0]][r], grid.cv_results_[cv_keys[1]][r] / 2.0, grid.cv_results_[cv_keys[2]][r])) if r > 10: break print('...') print('Best parameters: %s' % grid.best_params_) print('Accuracy: %.2f' % grid.best_score_) 0.632 +/- 0.09 {'lasso__alpha': 0.2, 'meta_regressor__n_estimators': 10, 'ridge__alpha': 0.05} 0.645 +/- 0.08 {'lasso__alpha': 0.2, 'meta_regressor__n_estimators': 10, 'ridge__alpha': 0.1} 0.641 +/- 0.08 {'lasso__alpha': 0.2, 'meta_regressor__n_estimators': 10, 'ridge__alpha': 0.15} 0.653 +/- 0.08 {'lasso__alpha': 0.2, 'meta_regressor__n_estimators': 10, 'ridge__alpha': 0.2} 0.622 +/- 0.10 {'lasso__alpha': 0.2, 'meta_regressor__n_estimators': 10, 'ridge__alpha': 0.25} 0.630 +/- 0.09 {'lasso__alpha': 0.2, 'meta_regressor__n_estimators': 10, 'ridge__alpha': 0.3} 0.630 +/- 0.09 {'lasso__alpha': 0.2, 'meta_regressor__n_estimators': 10, 'ridge__alpha': 0.35} 0.642 +/- 0.09 {'lasso__alpha': 0.2, 'meta_regressor__n_estimators': 10, 'ridge__alpha': 0.4} 0.654 +/- 0.08 {'lasso__alpha': 0.2, 'meta_regressor__n_estimators': 10, 'ridge__alpha': 0.45} 0.642 +/- 0.09 {'lasso__alpha': 0.2, 'meta_regressor__n_estimators': 100, 'ridge__alpha': 0.05} 0.645 +/- 0.09 {'lasso__alpha': 0.2, 'meta_regressor__n_estimators': 100, 'ridge__alpha': 0.1} 0.648 +/- 0.09 {'lasso__alpha': 0.2, 'meta_regressor__n_estimators': 100, 'ridge__alpha': 0.15} ... Best parameters: {'lasso__alpha': 1.6, 'meta_regressor__n_estimators': 10, 'ridge__alpha': 0.4} Accuracy: 0.68 Note The StackingCVRegressor also enables grid search over the regressors and even a single base regressor. When there are level-mixed hyperparameters, GridSearchCV will try to replace hyperparameters in a top-down order, i.e., regressors -> single base regressor -> regressor hyperparameter. For instance, given a hyperparameter grid such as params = {'randomforestregressor__n_estimators': [1, 100], 'regressors': [(regr1, regr1, regr1), (regr2, regr3)]} it will first use the instance settings of either (regr1, regr2, regr3) or (regr2, regr3) . Then it will replace the 'n_estimators' settings for a matching regressor based on 'randomforestregressor__n_estimators': [1, 100] . API StackingCVRegressor(regressors, meta_regressor, cv=5, shuffle=True, random_state=None, verbose=0, refit=True, use_features_in_secondary=False, store_train_meta_features=False, n_jobs=None, pre_dispatch='2 n_jobs', multi_output=False)* A 'Stacking Cross-Validation' regressor for scikit-learn estimators. Parameters regressors : array-like, shape = [n_regressors] A list of regressors. Invoking the fit method on the StackingCVRegressor will fit clones of these original regressors that will be stored in the class attribute self.regr_ . meta_regressor : object The meta-regressor to be fitted on the ensemble of regressor cv : int, cross-validation generator or iterable, optional (default: 5) Determines the cross-validation splitting strategy. Possible inputs for cv are: - None, to use the default 5-fold cross validation, - integer, to specify the number of folds in a KFold , - An object to be used as a cross-validation generator. - An iterable yielding train, test splits. For integer/None inputs, it will use KFold cross-validation shuffle : bool (default: True) If True, and the cv argument is integer, the training data will be shuffled at fitting stage prior to cross-validation. If the cv argument is a specific cross validation technique, this argument is omitted. random_state : int, RandomState instance or None, optional (default: None) Constrols the randomness of the cv splitter. Used when cv is integer and shuffle=True . New in v0.16.0. verbose : int, optional (default=0) Controls the verbosity of the building process. New in v0.16.0 refit : bool (default: True) Clones the regressors for stacking regression if True (default) or else uses the original ones, which will be refitted on the dataset upon calling the fit method. Setting refit=False is recommended if you are working with estimators that are supporting the scikit-learn fit/predict API interface but are not compatible to scikit-learn's clone function. use_features_in_secondary : bool (default: False) If True, the meta-regressor will be trained both on the predictions of the original regressors and the original dataset. If False, the meta-regressor will be trained only on the predictions of the original regressors. store_train_meta_features : bool (default: False) If True, the meta-features computed from the training data used for fitting the meta-regressor stored in the self.train_meta_features_ array, which can be accessed after calling fit . n_jobs : int or None, optional (default=None) The number of CPUs to use to do the computation. None means 1 unless in a :obj: joblib.parallel_backend context. -1 means using all processors. See :term: Glossary <n_jobs> for more details. New in v0.16.0. pre_dispatch : int, or string, optional Controls the number of jobs that get dispatched during parallel execution. Reducing this number can be useful to avoid an explosion of memory consumption when more jobs get dispatched than CPUs can process. This parameter can be: - None, in which case all the jobs are immediately created and spawned. Use this for lightweight and fast-running jobs, to avoid delays due to on-demand spawning of the jobs - An int, giving the exact number of total jobs that are spawned - A string, giving an expression as a function of n_jobs, as in '2*n_jobs' multi_output : bool (default: False) If True, allow multi-output targets, but forbid nan or inf values. If False, y will be checked to be a vector. (New in v0.19.0.) Attributes train_meta_features : numpy array, shape = [n_samples, n_regressors] meta-features for training data, where n_samples is the number of samples in training data and len(self.regressors) is the number of regressors. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/regressor/StackingCVRegressor/ Methods fit(X, y, groups=None, sample_weight=None) Fit ensemble regressors and the meta-regressor. Parameters X : numpy array, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : numpy array, shape = [n_samples] or [n_samples, n_targets] Target values. Multiple targets are supported only if self.multi_output is True. groups : numpy array/None, shape = [n_samples] The group that each sample belongs to. This is used by specific folding strategies such as GroupKFold() sample_weight : array-like, shape = [n_samples], optional Sample weights passed as sample_weights to each regressor in the regressors list as well as the meta_regressor. Raises error if some regressor does not support sample_weight in the fit() method. Returns self : object fit_transform(X, y=None, fit_params) Fit to data, then transform it. Fits transformer to `X` and `y` with optional parameters `fit_params` and returns a transformed version of `X`. Parameters X : array-like of shape (n_samples, n_features) Input samples. y : array-like of shape (n_samples,) or (n_samples, n_outputs), default=None Target values (None for unsupervised transformations). **fit_params : dict Additional fit parameters. Returns X_new : ndarray array of shape (n_samples, n_features_new) Transformed array. get_params(deep=True) Get parameters for this estimator. Parameters deep : bool, default=True If True, will return the parameters for this estimator and contained subobjects that are estimators. Returns params : dict Parameter names mapped to their values. predict(X) Predict target values for X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns y_target : array-like, shape = [n_samples] or [n_samples, n_targets] Predicted target values. predict_meta_features(X) Get meta-features of test-data. Parameters X : numpy array, shape = [n_samples, n_features] Test vectors, where n_samples is the number of samples and n_features is the number of features. Returns meta-features : numpy array, shape = [n_samples, len(self.regressors)] meta-features for test data, where n_samples is the number of samples in test data and len(self.regressors) is the number of regressors. If self.multi_output is True, then the number of columns is len(self.regressors) * n_targets. score(X, y, sample_weight=None) Return the coefficient of determination :math: R^2 of the prediction. The coefficient :math:`R^2` is defined as :math:`(1 - \\frac{u}{v})`, where :math:`u` is the residual sum of squares ``((y_true - y_pred) 2).sum() and :math:`v` is the total sum of squares ((y_true - y_true.mean()) 2).sum()``. The best possible score is 1.0 and it can be negative (because the model can be arbitrarily worse). A constant model that always predicts the expected value of y , disregarding the input features, would get a :math: R^2 score of 0.0. Parameters X : array-like of shape (n_samples, n_features) Test samples. For some estimators this may be a precomputed kernel matrix or a list of generic objects instead with shape (n_samples, n_samples_fitted) , where n_samples_fitted is the number of samples used in the fitting for the estimator. y : array-like of shape (n_samples,) or (n_samples, n_outputs) True values for X . sample_weight : array-like of shape (n_samples,), default=None Sample weights. Returns score : float :math: R^2 of self.predict(X) wrt. y . Notes The :math: R^2 score used when calling score on a regressor uses multioutput='uniform_average' from version 0.23 to keep consistent with default value of :func: ~sklearn.metrics.r2_score . This influences the score method of all the multioutput regressors (except for :class: ~sklearn.multioutput.MultiOutputRegressor ). set_params( params) Set the parameters of this estimator. Valid parameter keys can be listed with ``get_params()``. Returns self Properties named_regressors Returns List of named estimator tuples, like [('svc', SVC(...))]","title":"StackingCVRegressor: stacking with cross-validation for regression"},{"location":"user_guide/regressor/StackingCVRegressor/#stackingcvregressor-stacking-with-cross-validation-for-regression","text":"An ensemble-learning meta-regressor for stacking regression from mlxtend.regressor import StackingCVRegressor","title":"StackingCVRegressor: stacking with cross-validation for regression"},{"location":"user_guide/regressor/StackingCVRegressor/#overview","text":"Stacking is an ensemble learning technique to combine multiple regression models via a meta-regressor. The StackingCVRegressor extends the standard stacking algorithm (implemented as StackingRegressor ) using out-of-fold predictions to prepare the input data for the level-2 regressor. In the standard stacking procedure, the first-level regressors are fit to the same training set that is used prepare the inputs for the second-level regressor, which may lead to overfitting. The StackingCVRegressor , however, uses the concept of out-of-fold predictions: the dataset is split into k folds, and in k successive rounds, k-1 folds are used to fit the first level regressor. In each round, the first-level regressors are then applied to the remaining 1 subset that was not used for model fitting in each iteration. The resulting predictions are then stacked and provided -- as input data -- to the second-level regressor. After the training of the StackingCVRegressor , the first-level regressors are fit to the entire dataset for optimal predicitons.","title":"Overview"},{"location":"user_guide/regressor/StackingCVRegressor/#references","text":"Breiman, Leo. \" Stacked regressions. \" Machine learning 24.1 (1996): 49-64. Analogous implementation: StackingCVClassifier","title":"References"},{"location":"user_guide/regressor/StackingCVRegressor/#example-1-boston-housing-data-predictions","text":"In this example we evaluate some basic prediction models on the boston housing dataset and see how the R^2 and MSE scores are affected by combining the models with StackingCVRegressor . The code output below demonstrates that the stacked model performs the best on this dataset -- slightly better than the best single regression model. from mlxtend.regressor import StackingCVRegressor from sklearn.datasets import load_boston from sklearn.svm import SVR from sklearn.linear_model import Lasso from sklearn.ensemble import RandomForestRegressor from sklearn.model_selection import cross_val_score import numpy as np RANDOM_SEED = 42 X, y = load_boston(return_X_y=True) svr = SVR(kernel='linear') lasso = Lasso() rf = RandomForestRegressor(n_estimators=5, random_state=RANDOM_SEED) # Starting from v0.16.0, StackingCVRegressor supports # `random_state` to get deterministic result. stack = StackingCVRegressor(regressors=(svr, lasso, rf), meta_regressor=lasso, random_state=RANDOM_SEED) print('5-fold cross validation scores:\\n') for clf, label in zip([svr, lasso, rf, stack], ['SVM', 'Lasso', 'Random Forest', 'StackingCVRegressor']): scores = cross_val_score(clf, X, y, cv=5) print(\"R^2 Score: %0.2f (+/- %0.2f) [%s]\" % ( scores.mean(), scores.std(), label)) 5-fold cross validation scores: R^2 Score: 0.46 (+/- 0.29) [SVM] R^2 Score: 0.43 (+/- 0.14) [Lasso] R^2 Score: 0.53 (+/- 0.28) [Random Forest] R^2 Score: 0.57 (+/- 0.24) [StackingCVRegressor] stack = StackingCVRegressor(regressors=(svr, lasso, rf), meta_regressor=lasso) print('5-fold cross validation scores:\\n') for clf, label in zip([svr, lasso, rf, stack], ['SVM', 'Lasso', 'Random Forest', 'StackingCVRegressor']): scores = cross_val_score(clf, X, y, cv=5, scoring='neg_mean_squared_error') print(\"Neg. MSE Score: %0.2f (+/- %0.2f) [%s]\" % ( scores.mean(), scores.std(), label)) 5-fold cross validation scores: Neg. MSE Score: -33.33 (+/- 22.36) [SVM] Neg. MSE Score: -35.53 (+/- 16.99) [Lasso] Neg. MSE Score: -27.08 (+/- 15.67) [Random Forest] Neg. MSE Score: -25.85 (+/- 17.22) [StackingCVRegressor]","title":"Example 1: Boston Housing Data Predictions"},{"location":"user_guide/regressor/StackingCVRegressor/#example-2-gridsearchcv-with-stacking","text":"In this second example we demonstrate how StackingCVRegressor works in combination with GridSearchCV . The stack still allows tuning hyper parameters of the base and meta models! For instance, we can use estimator.get_params().keys() to get a full list of tunable parameters. from mlxtend.regressor import StackingCVRegressor from sklearn.datasets import load_boston from sklearn.linear_model import Lasso from sklearn.linear_model import Ridge from sklearn.ensemble import RandomForestRegressor from sklearn.model_selection import GridSearchCV X, y = load_boston(return_X_y=True) ridge = Ridge(random_state=RANDOM_SEED) lasso = Lasso(random_state=RANDOM_SEED) rf = RandomForestRegressor(random_state=RANDOM_SEED) stack = StackingCVRegressor(regressors=(lasso, ridge), meta_regressor=rf, random_state=RANDOM_SEED, use_features_in_secondary=True) params = {'lasso__alpha': [0.1, 1.0, 10.0], 'ridge__alpha': [0.1, 1.0, 10.0]} grid = GridSearchCV( estimator=stack, param_grid={ 'lasso__alpha': [x/5.0 for x in range(1, 10)], 'ridge__alpha': [x/20.0 for x in range(1, 10)], 'meta_regressor__n_estimators': [10, 100] }, cv=5, refit=True ) grid.fit(X, y) print(\"Best: %f using %s\" % (grid.best_score_, grid.best_params_)) Best: 0.679151 using {'lasso__alpha': 1.6, 'meta_regressor__n_estimators': 10, 'ridge__alpha': 0.4} cv_keys = ('mean_test_score', 'std_test_score', 'params') for r, _ in enumerate(grid.cv_results_['mean_test_score']): print(\"%0.3f +/- %0.2f %r\" % (grid.cv_results_[cv_keys[0]][r], grid.cv_results_[cv_keys[1]][r] / 2.0, grid.cv_results_[cv_keys[2]][r])) if r > 10: break print('...') print('Best parameters: %s' % grid.best_params_) print('Accuracy: %.2f' % grid.best_score_) 0.632 +/- 0.09 {'lasso__alpha': 0.2, 'meta_regressor__n_estimators': 10, 'ridge__alpha': 0.05} 0.645 +/- 0.08 {'lasso__alpha': 0.2, 'meta_regressor__n_estimators': 10, 'ridge__alpha': 0.1} 0.641 +/- 0.08 {'lasso__alpha': 0.2, 'meta_regressor__n_estimators': 10, 'ridge__alpha': 0.15} 0.653 +/- 0.08 {'lasso__alpha': 0.2, 'meta_regressor__n_estimators': 10, 'ridge__alpha': 0.2} 0.622 +/- 0.10 {'lasso__alpha': 0.2, 'meta_regressor__n_estimators': 10, 'ridge__alpha': 0.25} 0.630 +/- 0.09 {'lasso__alpha': 0.2, 'meta_regressor__n_estimators': 10, 'ridge__alpha': 0.3} 0.630 +/- 0.09 {'lasso__alpha': 0.2, 'meta_regressor__n_estimators': 10, 'ridge__alpha': 0.35} 0.642 +/- 0.09 {'lasso__alpha': 0.2, 'meta_regressor__n_estimators': 10, 'ridge__alpha': 0.4} 0.654 +/- 0.08 {'lasso__alpha': 0.2, 'meta_regressor__n_estimators': 10, 'ridge__alpha': 0.45} 0.642 +/- 0.09 {'lasso__alpha': 0.2, 'meta_regressor__n_estimators': 100, 'ridge__alpha': 0.05} 0.645 +/- 0.09 {'lasso__alpha': 0.2, 'meta_regressor__n_estimators': 100, 'ridge__alpha': 0.1} 0.648 +/- 0.09 {'lasso__alpha': 0.2, 'meta_regressor__n_estimators': 100, 'ridge__alpha': 0.15} ... Best parameters: {'lasso__alpha': 1.6, 'meta_regressor__n_estimators': 10, 'ridge__alpha': 0.4} Accuracy: 0.68 Note The StackingCVRegressor also enables grid search over the regressors and even a single base regressor. When there are level-mixed hyperparameters, GridSearchCV will try to replace hyperparameters in a top-down order, i.e., regressors -> single base regressor -> regressor hyperparameter. For instance, given a hyperparameter grid such as params = {'randomforestregressor__n_estimators': [1, 100], 'regressors': [(regr1, regr1, regr1), (regr2, regr3)]} it will first use the instance settings of either (regr1, regr2, regr3) or (regr2, regr3) . Then it will replace the 'n_estimators' settings for a matching regressor based on 'randomforestregressor__n_estimators': [1, 100] .","title":"Example 2: GridSearchCV with Stacking"},{"location":"user_guide/regressor/StackingCVRegressor/#api","text":"StackingCVRegressor(regressors, meta_regressor, cv=5, shuffle=True, random_state=None, verbose=0, refit=True, use_features_in_secondary=False, store_train_meta_features=False, n_jobs=None, pre_dispatch='2 n_jobs', multi_output=False)* A 'Stacking Cross-Validation' regressor for scikit-learn estimators. Parameters regressors : array-like, shape = [n_regressors] A list of regressors. Invoking the fit method on the StackingCVRegressor will fit clones of these original regressors that will be stored in the class attribute self.regr_ . meta_regressor : object The meta-regressor to be fitted on the ensemble of regressor cv : int, cross-validation generator or iterable, optional (default: 5) Determines the cross-validation splitting strategy. Possible inputs for cv are: - None, to use the default 5-fold cross validation, - integer, to specify the number of folds in a KFold , - An object to be used as a cross-validation generator. - An iterable yielding train, test splits. For integer/None inputs, it will use KFold cross-validation shuffle : bool (default: True) If True, and the cv argument is integer, the training data will be shuffled at fitting stage prior to cross-validation. If the cv argument is a specific cross validation technique, this argument is omitted. random_state : int, RandomState instance or None, optional (default: None) Constrols the randomness of the cv splitter. Used when cv is integer and shuffle=True . New in v0.16.0. verbose : int, optional (default=0) Controls the verbosity of the building process. New in v0.16.0 refit : bool (default: True) Clones the regressors for stacking regression if True (default) or else uses the original ones, which will be refitted on the dataset upon calling the fit method. Setting refit=False is recommended if you are working with estimators that are supporting the scikit-learn fit/predict API interface but are not compatible to scikit-learn's clone function. use_features_in_secondary : bool (default: False) If True, the meta-regressor will be trained both on the predictions of the original regressors and the original dataset. If False, the meta-regressor will be trained only on the predictions of the original regressors. store_train_meta_features : bool (default: False) If True, the meta-features computed from the training data used for fitting the meta-regressor stored in the self.train_meta_features_ array, which can be accessed after calling fit . n_jobs : int or None, optional (default=None) The number of CPUs to use to do the computation. None means 1 unless in a :obj: joblib.parallel_backend context. -1 means using all processors. See :term: Glossary <n_jobs> for more details. New in v0.16.0. pre_dispatch : int, or string, optional Controls the number of jobs that get dispatched during parallel execution. Reducing this number can be useful to avoid an explosion of memory consumption when more jobs get dispatched than CPUs can process. This parameter can be: - None, in which case all the jobs are immediately created and spawned. Use this for lightweight and fast-running jobs, to avoid delays due to on-demand spawning of the jobs - An int, giving the exact number of total jobs that are spawned - A string, giving an expression as a function of n_jobs, as in '2*n_jobs' multi_output : bool (default: False) If True, allow multi-output targets, but forbid nan or inf values. If False, y will be checked to be a vector. (New in v0.19.0.) Attributes train_meta_features : numpy array, shape = [n_samples, n_regressors] meta-features for training data, where n_samples is the number of samples in training data and len(self.regressors) is the number of regressors. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/regressor/StackingCVRegressor/","title":"API"},{"location":"user_guide/regressor/StackingCVRegressor/#methods","text":"fit(X, y, groups=None, sample_weight=None) Fit ensemble regressors and the meta-regressor. Parameters X : numpy array, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : numpy array, shape = [n_samples] or [n_samples, n_targets] Target values. Multiple targets are supported only if self.multi_output is True. groups : numpy array/None, shape = [n_samples] The group that each sample belongs to. This is used by specific folding strategies such as GroupKFold() sample_weight : array-like, shape = [n_samples], optional Sample weights passed as sample_weights to each regressor in the regressors list as well as the meta_regressor. Raises error if some regressor does not support sample_weight in the fit() method. Returns self : object fit_transform(X, y=None, fit_params) Fit to data, then transform it. Fits transformer to `X` and `y` with optional parameters `fit_params` and returns a transformed version of `X`. Parameters X : array-like of shape (n_samples, n_features) Input samples. y : array-like of shape (n_samples,) or (n_samples, n_outputs), default=None Target values (None for unsupervised transformations). **fit_params : dict Additional fit parameters. Returns X_new : ndarray array of shape (n_samples, n_features_new) Transformed array. get_params(deep=True) Get parameters for this estimator. Parameters deep : bool, default=True If True, will return the parameters for this estimator and contained subobjects that are estimators. Returns params : dict Parameter names mapped to their values. predict(X) Predict target values for X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns y_target : array-like, shape = [n_samples] or [n_samples, n_targets] Predicted target values. predict_meta_features(X) Get meta-features of test-data. Parameters X : numpy array, shape = [n_samples, n_features] Test vectors, where n_samples is the number of samples and n_features is the number of features. Returns meta-features : numpy array, shape = [n_samples, len(self.regressors)] meta-features for test data, where n_samples is the number of samples in test data and len(self.regressors) is the number of regressors. If self.multi_output is True, then the number of columns is len(self.regressors) * n_targets. score(X, y, sample_weight=None) Return the coefficient of determination :math: R^2 of the prediction. The coefficient :math:`R^2` is defined as :math:`(1 - \\frac{u}{v})`, where :math:`u` is the residual sum of squares ``((y_true - y_pred) 2).sum() and :math:`v` is the total sum of squares ((y_true - y_true.mean()) 2).sum()``. The best possible score is 1.0 and it can be negative (because the model can be arbitrarily worse). A constant model that always predicts the expected value of y , disregarding the input features, would get a :math: R^2 score of 0.0. Parameters X : array-like of shape (n_samples, n_features) Test samples. For some estimators this may be a precomputed kernel matrix or a list of generic objects instead with shape (n_samples, n_samples_fitted) , where n_samples_fitted is the number of samples used in the fitting for the estimator. y : array-like of shape (n_samples,) or (n_samples, n_outputs) True values for X . sample_weight : array-like of shape (n_samples,), default=None Sample weights. Returns score : float :math: R^2 of self.predict(X) wrt. y . Notes The :math: R^2 score used when calling score on a regressor uses multioutput='uniform_average' from version 0.23 to keep consistent with default value of :func: ~sklearn.metrics.r2_score . This influences the score method of all the multioutput regressors (except for :class: ~sklearn.multioutput.MultiOutputRegressor ). set_params( params) Set the parameters of this estimator. Valid parameter keys can be listed with ``get_params()``. Returns self","title":"Methods"},{"location":"user_guide/regressor/StackingCVRegressor/#properties","text":"named_regressors Returns List of named estimator tuples, like [('svc', SVC(...))]","title":"Properties"},{"location":"user_guide/regressor/StackingRegressor/","text":"StackingRegressor: a simple stacking implementation for regression An ensemble-learning meta-regressor for stacking regression from mlxtend.regressor import StackingRegressor Overview Stacking regression is an ensemble learning technique to combine multiple regression models via a meta-regressor. The individual regression models are trained based on the complete training set; then, the meta-regressor is fitted based on the outputs -- meta-features -- of the individual regression models in the ensemble. References Breiman, Leo. \" Stacked regressions. \" Machine learning 24.1 (1996): 49-64. Example 1 - Simple Stacked Regression from mlxtend.regressor import StackingRegressor from mlxtend.data import boston_housing_data from sklearn.linear_model import LinearRegression from sklearn.linear_model import Ridge from sklearn.svm import SVR import matplotlib.pyplot as plt import numpy as np import warnings warnings.simplefilter('ignore') # Generating a sample dataset np.random.seed(1) X = np.sort(5 * np.random.rand(40, 1), axis=0) y = np.sin(X).ravel() y[::5] += 3 * (0.5 - np.random.rand(8)) # Initializing models lr = LinearRegression() svr_lin = SVR(kernel='linear') ridge = Ridge(random_state=1) svr_rbf = SVR(kernel='rbf') stregr = StackingRegressor(regressors=[svr_lin, lr, ridge], meta_regressor=svr_rbf) # Training the stacking classifier stregr.fit(X, y) stregr.predict(X) # Evaluate and visualize the fit print(\"Mean Squared Error: %.4f\" % np.mean((stregr.predict(X) - y) ** 2)) print('Variance Score: %.4f' % stregr.score(X, y)) with plt.style.context(('seaborn-whitegrid')): plt.scatter(X, y, c='lightgray') plt.plot(X, stregr.predict(X), c='darkgreen', lw=2) plt.show() Mean Squared Error: 0.1846 Variance Score: 0.7329 stregr StackingRegressor(meta_regressor=SVR(), regressors=[SVR(kernel='linear'), LinearRegression(), Ridge(random_state=1)]) Example 2 - Stacked Regression and GridSearch In this second example we demonstrate how StackingCVRegressor works in combination with GridSearchCV . The stack still allows tuning hyper parameters of the base and meta models! For instance, we can use estimator.get_params().keys() to get a full list of tunable parameters. from sklearn.model_selection import GridSearchCV from sklearn.linear_model import Lasso # Initializing models lr = LinearRegression() svr_lin = SVR(kernel='linear') ridge = Ridge(random_state=1) lasso = Lasso(random_state=1) svr_rbf = SVR(kernel='rbf') regressors = [svr_lin, lr, ridge, lasso] stregr = StackingRegressor(regressors=regressors, meta_regressor=svr_rbf) params = {'lasso__alpha': [0.1, 1.0, 10.0], 'ridge__alpha': [0.1, 1.0, 10.0], 'svr__C': [0.1, 1.0, 10.0], 'meta_regressor__C': [0.1, 1.0, 10.0, 100.0], 'meta_regressor__gamma': [0.1, 1.0, 10.0]} grid = GridSearchCV(estimator=stregr, param_grid=params, cv=5, refit=True) grid.fit(X, y) print(\"Best: %f using %s\" % (grid.best_score_, grid.best_params_)) Best: -0.082717 using {'lasso__alpha': 0.1, 'meta_regressor__C': 1.0, 'meta_regressor__gamma': 1.0, 'ridge__alpha': 0.1, 'svr__C': 10.0} cv_keys = ('mean_test_score', 'std_test_score', 'params') for r, _ in enumerate(grid.cv_results_['mean_test_score']): print(\"%0.3f +/- %0.2f %r\" % (grid.cv_results_[cv_keys[0]][r], grid.cv_results_[cv_keys[1]][r] / 2.0, grid.cv_results_[cv_keys[2]][r])) if r > 10: break print('...') print('Best parameters: %s' % grid.best_params_) print('Accuracy: %.2f' % grid.best_score_) -9.810 +/- 6.86 {'lasso__alpha': 0.1, 'meta_regressor__C': 0.1, 'meta_regressor__gamma': 0.1, 'ridge__alpha': 0.1, 'svr__C': 0.1} -9.591 +/- 6.67 {'lasso__alpha': 0.1, 'meta_regressor__C': 0.1, 'meta_regressor__gamma': 0.1, 'ridge__alpha': 0.1, 'svr__C': 1.0} -9.591 +/- 6.67 {'lasso__alpha': 0.1, 'meta_regressor__C': 0.1, 'meta_regressor__gamma': 0.1, 'ridge__alpha': 0.1, 'svr__C': 10.0} -9.819 +/- 6.87 {'lasso__alpha': 0.1, 'meta_regressor__C': 0.1, 'meta_regressor__gamma': 0.1, 'ridge__alpha': 1.0, 'svr__C': 0.1} -9.600 +/- 6.68 {'lasso__alpha': 0.1, 'meta_regressor__C': 0.1, 'meta_regressor__gamma': 0.1, 'ridge__alpha': 1.0, 'svr__C': 1.0} -9.600 +/- 6.68 {'lasso__alpha': 0.1, 'meta_regressor__C': 0.1, 'meta_regressor__gamma': 0.1, 'ridge__alpha': 1.0, 'svr__C': 10.0} -9.878 +/- 6.91 {'lasso__alpha': 0.1, 'meta_regressor__C': 0.1, 'meta_regressor__gamma': 0.1, 'ridge__alpha': 10.0, 'svr__C': 0.1} -9.665 +/- 6.71 {'lasso__alpha': 0.1, 'meta_regressor__C': 0.1, 'meta_regressor__gamma': 0.1, 'ridge__alpha': 10.0, 'svr__C': 1.0} -9.665 +/- 6.71 {'lasso__alpha': 0.1, 'meta_regressor__C': 0.1, 'meta_regressor__gamma': 0.1, 'ridge__alpha': 10.0, 'svr__C': 10.0} -4.839 +/- 3.98 {'lasso__alpha': 0.1, 'meta_regressor__C': 0.1, 'meta_regressor__gamma': 1.0, 'ridge__alpha': 0.1, 'svr__C': 0.1} -3.986 +/- 3.16 {'lasso__alpha': 0.1, 'meta_regressor__C': 0.1, 'meta_regressor__gamma': 1.0, 'ridge__alpha': 0.1, 'svr__C': 1.0} -3.986 +/- 3.16 {'lasso__alpha': 0.1, 'meta_regressor__C': 0.1, 'meta_regressor__gamma': 1.0, 'ridge__alpha': 0.1, 'svr__C': 10.0} ... Best parameters: {'lasso__alpha': 0.1, 'meta_regressor__C': 1.0, 'meta_regressor__gamma': 1.0, 'ridge__alpha': 0.1, 'svr__C': 10.0} Accuracy: -0.08 # Evaluate and visualize the fit print(\"Mean Squared Error: %.4f\" % np.mean((grid.predict(X) - y) ** 2)) print('Variance Score: %.4f' % grid.score(X, y)) with plt.style.context(('seaborn-whitegrid')): plt.scatter(X, y, c='lightgray') plt.plot(X, grid.predict(X), c='darkgreen', lw=2) plt.show() Mean Squared Error: 0.1845 Variance Score: 0.7330 Note The StackingCVRegressor also enables grid search over the regressors and even a single base regressor. When there are level-mixed hyperparameters, GridSearchCV will try to replace hyperparameters in a top-down order, i.e., regressors -> single base regressor -> regressor hyperparameter. For instance, given a hyperparameter grid such as params = {'randomforestregressor__n_estimators': [1, 100], 'regressors': [(regr1, regr1, regr1), (regr2, regr3)]} it will first use the instance settings of either (regr1, regr2, regr3) or (regr2, regr3) . Then it will replace the 'n_estimators' settings for a matching regressor based on 'randomforestregressor__n_estimators': [1, 100] . API StackingRegressor(regressors, meta_regressor, verbose=0, use_features_in_secondary=False, store_train_meta_features=False, refit=True, multi_output=False) A Stacking regressor for scikit-learn estimators for regression. Parameters regressors : array-like, shape = [n_regressors] A list of regressors. Invoking the fit method on the StackingRegressor will fit clones of those original regressors that will be stored in the class attribute self.regr_ . meta_regressor : object The meta-regressor to be fitted on the ensemble of regressors verbose : int, optional (default=0) Controls the verbosity of the building process. - verbose=0 (default): Prints nothing - verbose=1 : Prints the number & name of the regressor being fitted - verbose=2 : Prints info about the parameters of the regressor being fitted - verbose>2 : Changes verbose param of the underlying regressor to self.verbose - 2 use_features_in_secondary : bool (default: False) If True, the meta-regressor will be trained both on the predictions of the original regressors and the original dataset. If False, the meta-regressor will be trained only on the predictions of the original regressors. store_train_meta_features : bool (default: False) If True, the meta-features computed from the training data used for fitting the meta-regressor stored in the self.train_meta_features_ array, which can be accessed after calling fit . Attributes regr_ : list, shape=[n_regressors] Fitted regressors (clones of the original regressors) meta_regr_ : estimator Fitted meta-regressor (clone of the original meta-estimator) coef_ : array-like, shape = [n_features] Model coefficients of the fitted meta-estimator intercept_ : float Intercept of the fitted meta-estimator train_meta_features : numpy array, shape = [n_samples, len(self.regressors)] meta-features for training data, where n_samples is the number of samples in training data and len(self.regressors) is the number of regressors. refit : bool (default: True) Clones the regressors for stacking regression if True (default) or else uses the original ones, which will be refitted on the dataset upon calling the fit method. Setting refit=False is recommended if you are working with estimators that are supporting the scikit-learn fit/predict API interface but are not compatible to scikit-learn's clone function. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/regressor/StackingRegressor/ Methods fit(X, y, sample_weight=None) Learn weight coefficients from training data for each regressor. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : numpy array, shape = [n_samples] or [n_samples, n_targets] Target values. Multiple targets are supported only if self.multi_output is True. sample_weight : array-like, shape = [n_samples], optional Sample weights passed as sample_weights to each regressor in the regressors list as well as the meta_regressor. Raises error if some regressor does not support sample_weight in the fit() method. Returns self : object fit_transform(X, y=None, fit_params) Fit to data, then transform it. Fits transformer to `X` and `y` with optional parameters `fit_params` and returns a transformed version of `X`. Parameters X : array-like of shape (n_samples, n_features) Input samples. y : array-like of shape (n_samples,) or (n_samples, n_outputs), default=None Target values (None for unsupervised transformations). **fit_params : dict Additional fit parameters. Returns X_new : ndarray array of shape (n_samples, n_features_new) Transformed array. get_params(deep=True) Return estimator parameter names for GridSearch support. predict(X) Predict target values for X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns y_target : array-like, shape = [n_samples] or [n_samples, n_targets] Predicted target values. predict_meta_features(X) Get meta-features of test-data. Parameters X : numpy array, shape = [n_samples, n_features] Test vectors, where n_samples is the number of samples and n_features is the number of features. Returns meta-features : numpy array, shape = [n_samples, len(self.regressors)] meta-features for test data, where n_samples is the number of samples in test data and len(self.regressors) is the number of regressors. If self.multi_output is True, then the number of columns is len(self.regressors) * n_targets score(X, y, sample_weight=None) Return the coefficient of determination :math: R^2 of the prediction. The coefficient :math:`R^2` is defined as :math:`(1 - \\frac{u}{v})`, where :math:`u` is the residual sum of squares ``((y_true - y_pred) 2).sum() and :math:`v` is the total sum of squares ((y_true - y_true.mean()) 2).sum()``. The best possible score is 1.0 and it can be negative (because the model can be arbitrarily worse). A constant model that always predicts the expected value of y , disregarding the input features, would get a :math: R^2 score of 0.0. Parameters X : array-like of shape (n_samples, n_features) Test samples. For some estimators this may be a precomputed kernel matrix or a list of generic objects instead with shape (n_samples, n_samples_fitted) , where n_samples_fitted is the number of samples used in the fitting for the estimator. y : array-like of shape (n_samples,) or (n_samples, n_outputs) True values for X . sample_weight : array-like of shape (n_samples,), default=None Sample weights. Returns score : float :math: R^2 of self.predict(X) wrt. y . Notes The :math: R^2 score used when calling score on a regressor uses multioutput='uniform_average' from version 0.23 to keep consistent with default value of :func: ~sklearn.metrics.r2_score . This influences the score method of all the multioutput regressors (except for :class: ~sklearn.multioutput.MultiOutputRegressor ). set_params( params) Set the parameters of this estimator. Valid parameter keys can be listed with ``get_params()``. Returns self Properties coef_ None intercept_ None named_regressors None ython","title":"StackingRegressor: a simple stacking implementation for regression"},{"location":"user_guide/regressor/StackingRegressor/#stackingregressor-a-simple-stacking-implementation-for-regression","text":"An ensemble-learning meta-regressor for stacking regression from mlxtend.regressor import StackingRegressor","title":"StackingRegressor: a simple stacking implementation for regression"},{"location":"user_guide/regressor/StackingRegressor/#overview","text":"Stacking regression is an ensemble learning technique to combine multiple regression models via a meta-regressor. The individual regression models are trained based on the complete training set; then, the meta-regressor is fitted based on the outputs -- meta-features -- of the individual regression models in the ensemble.","title":"Overview"},{"location":"user_guide/regressor/StackingRegressor/#references","text":"Breiman, Leo. \" Stacked regressions. \" Machine learning 24.1 (1996): 49-64.","title":"References"},{"location":"user_guide/regressor/StackingRegressor/#example-1-simple-stacked-regression","text":"from mlxtend.regressor import StackingRegressor from mlxtend.data import boston_housing_data from sklearn.linear_model import LinearRegression from sklearn.linear_model import Ridge from sklearn.svm import SVR import matplotlib.pyplot as plt import numpy as np import warnings warnings.simplefilter('ignore') # Generating a sample dataset np.random.seed(1) X = np.sort(5 * np.random.rand(40, 1), axis=0) y = np.sin(X).ravel() y[::5] += 3 * (0.5 - np.random.rand(8)) # Initializing models lr = LinearRegression() svr_lin = SVR(kernel='linear') ridge = Ridge(random_state=1) svr_rbf = SVR(kernel='rbf') stregr = StackingRegressor(regressors=[svr_lin, lr, ridge], meta_regressor=svr_rbf) # Training the stacking classifier stregr.fit(X, y) stregr.predict(X) # Evaluate and visualize the fit print(\"Mean Squared Error: %.4f\" % np.mean((stregr.predict(X) - y) ** 2)) print('Variance Score: %.4f' % stregr.score(X, y)) with plt.style.context(('seaborn-whitegrid')): plt.scatter(X, y, c='lightgray') plt.plot(X, stregr.predict(X), c='darkgreen', lw=2) plt.show() Mean Squared Error: 0.1846 Variance Score: 0.7329 stregr StackingRegressor(meta_regressor=SVR(), regressors=[SVR(kernel='linear'), LinearRegression(), Ridge(random_state=1)])","title":"Example 1 - Simple Stacked Regression"},{"location":"user_guide/regressor/StackingRegressor/#example-2-stacked-regression-and-gridsearch","text":"In this second example we demonstrate how StackingCVRegressor works in combination with GridSearchCV . The stack still allows tuning hyper parameters of the base and meta models! For instance, we can use estimator.get_params().keys() to get a full list of tunable parameters. from sklearn.model_selection import GridSearchCV from sklearn.linear_model import Lasso # Initializing models lr = LinearRegression() svr_lin = SVR(kernel='linear') ridge = Ridge(random_state=1) lasso = Lasso(random_state=1) svr_rbf = SVR(kernel='rbf') regressors = [svr_lin, lr, ridge, lasso] stregr = StackingRegressor(regressors=regressors, meta_regressor=svr_rbf) params = {'lasso__alpha': [0.1, 1.0, 10.0], 'ridge__alpha': [0.1, 1.0, 10.0], 'svr__C': [0.1, 1.0, 10.0], 'meta_regressor__C': [0.1, 1.0, 10.0, 100.0], 'meta_regressor__gamma': [0.1, 1.0, 10.0]} grid = GridSearchCV(estimator=stregr, param_grid=params, cv=5, refit=True) grid.fit(X, y) print(\"Best: %f using %s\" % (grid.best_score_, grid.best_params_)) Best: -0.082717 using {'lasso__alpha': 0.1, 'meta_regressor__C': 1.0, 'meta_regressor__gamma': 1.0, 'ridge__alpha': 0.1, 'svr__C': 10.0} cv_keys = ('mean_test_score', 'std_test_score', 'params') for r, _ in enumerate(grid.cv_results_['mean_test_score']): print(\"%0.3f +/- %0.2f %r\" % (grid.cv_results_[cv_keys[0]][r], grid.cv_results_[cv_keys[1]][r] / 2.0, grid.cv_results_[cv_keys[2]][r])) if r > 10: break print('...') print('Best parameters: %s' % grid.best_params_) print('Accuracy: %.2f' % grid.best_score_) -9.810 +/- 6.86 {'lasso__alpha': 0.1, 'meta_regressor__C': 0.1, 'meta_regressor__gamma': 0.1, 'ridge__alpha': 0.1, 'svr__C': 0.1} -9.591 +/- 6.67 {'lasso__alpha': 0.1, 'meta_regressor__C': 0.1, 'meta_regressor__gamma': 0.1, 'ridge__alpha': 0.1, 'svr__C': 1.0} -9.591 +/- 6.67 {'lasso__alpha': 0.1, 'meta_regressor__C': 0.1, 'meta_regressor__gamma': 0.1, 'ridge__alpha': 0.1, 'svr__C': 10.0} -9.819 +/- 6.87 {'lasso__alpha': 0.1, 'meta_regressor__C': 0.1, 'meta_regressor__gamma': 0.1, 'ridge__alpha': 1.0, 'svr__C': 0.1} -9.600 +/- 6.68 {'lasso__alpha': 0.1, 'meta_regressor__C': 0.1, 'meta_regressor__gamma': 0.1, 'ridge__alpha': 1.0, 'svr__C': 1.0} -9.600 +/- 6.68 {'lasso__alpha': 0.1, 'meta_regressor__C': 0.1, 'meta_regressor__gamma': 0.1, 'ridge__alpha': 1.0, 'svr__C': 10.0} -9.878 +/- 6.91 {'lasso__alpha': 0.1, 'meta_regressor__C': 0.1, 'meta_regressor__gamma': 0.1, 'ridge__alpha': 10.0, 'svr__C': 0.1} -9.665 +/- 6.71 {'lasso__alpha': 0.1, 'meta_regressor__C': 0.1, 'meta_regressor__gamma': 0.1, 'ridge__alpha': 10.0, 'svr__C': 1.0} -9.665 +/- 6.71 {'lasso__alpha': 0.1, 'meta_regressor__C': 0.1, 'meta_regressor__gamma': 0.1, 'ridge__alpha': 10.0, 'svr__C': 10.0} -4.839 +/- 3.98 {'lasso__alpha': 0.1, 'meta_regressor__C': 0.1, 'meta_regressor__gamma': 1.0, 'ridge__alpha': 0.1, 'svr__C': 0.1} -3.986 +/- 3.16 {'lasso__alpha': 0.1, 'meta_regressor__C': 0.1, 'meta_regressor__gamma': 1.0, 'ridge__alpha': 0.1, 'svr__C': 1.0} -3.986 +/- 3.16 {'lasso__alpha': 0.1, 'meta_regressor__C': 0.1, 'meta_regressor__gamma': 1.0, 'ridge__alpha': 0.1, 'svr__C': 10.0} ... Best parameters: {'lasso__alpha': 0.1, 'meta_regressor__C': 1.0, 'meta_regressor__gamma': 1.0, 'ridge__alpha': 0.1, 'svr__C': 10.0} Accuracy: -0.08 # Evaluate and visualize the fit print(\"Mean Squared Error: %.4f\" % np.mean((grid.predict(X) - y) ** 2)) print('Variance Score: %.4f' % grid.score(X, y)) with plt.style.context(('seaborn-whitegrid')): plt.scatter(X, y, c='lightgray') plt.plot(X, grid.predict(X), c='darkgreen', lw=2) plt.show() Mean Squared Error: 0.1845 Variance Score: 0.7330 Note The StackingCVRegressor also enables grid search over the regressors and even a single base regressor. When there are level-mixed hyperparameters, GridSearchCV will try to replace hyperparameters in a top-down order, i.e., regressors -> single base regressor -> regressor hyperparameter. For instance, given a hyperparameter grid such as params = {'randomforestregressor__n_estimators': [1, 100], 'regressors': [(regr1, regr1, regr1), (regr2, regr3)]} it will first use the instance settings of either (regr1, regr2, regr3) or (regr2, regr3) . Then it will replace the 'n_estimators' settings for a matching regressor based on 'randomforestregressor__n_estimators': [1, 100] .","title":"Example 2 - Stacked Regression and GridSearch"},{"location":"user_guide/regressor/StackingRegressor/#api","text":"StackingRegressor(regressors, meta_regressor, verbose=0, use_features_in_secondary=False, store_train_meta_features=False, refit=True, multi_output=False) A Stacking regressor for scikit-learn estimators for regression. Parameters regressors : array-like, shape = [n_regressors] A list of regressors. Invoking the fit method on the StackingRegressor will fit clones of those original regressors that will be stored in the class attribute self.regr_ . meta_regressor : object The meta-regressor to be fitted on the ensemble of regressors verbose : int, optional (default=0) Controls the verbosity of the building process. - verbose=0 (default): Prints nothing - verbose=1 : Prints the number & name of the regressor being fitted - verbose=2 : Prints info about the parameters of the regressor being fitted - verbose>2 : Changes verbose param of the underlying regressor to self.verbose - 2 use_features_in_secondary : bool (default: False) If True, the meta-regressor will be trained both on the predictions of the original regressors and the original dataset. If False, the meta-regressor will be trained only on the predictions of the original regressors. store_train_meta_features : bool (default: False) If True, the meta-features computed from the training data used for fitting the meta-regressor stored in the self.train_meta_features_ array, which can be accessed after calling fit . Attributes regr_ : list, shape=[n_regressors] Fitted regressors (clones of the original regressors) meta_regr_ : estimator Fitted meta-regressor (clone of the original meta-estimator) coef_ : array-like, shape = [n_features] Model coefficients of the fitted meta-estimator intercept_ : float Intercept of the fitted meta-estimator train_meta_features : numpy array, shape = [n_samples, len(self.regressors)] meta-features for training data, where n_samples is the number of samples in training data and len(self.regressors) is the number of regressors. refit : bool (default: True) Clones the regressors for stacking regression if True (default) or else uses the original ones, which will be refitted on the dataset upon calling the fit method. Setting refit=False is recommended if you are working with estimators that are supporting the scikit-learn fit/predict API interface but are not compatible to scikit-learn's clone function. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/regressor/StackingRegressor/","title":"API"},{"location":"user_guide/regressor/StackingRegressor/#methods","text":"fit(X, y, sample_weight=None) Learn weight coefficients from training data for each regressor. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : numpy array, shape = [n_samples] or [n_samples, n_targets] Target values. Multiple targets are supported only if self.multi_output is True. sample_weight : array-like, shape = [n_samples], optional Sample weights passed as sample_weights to each regressor in the regressors list as well as the meta_regressor. Raises error if some regressor does not support sample_weight in the fit() method. Returns self : object fit_transform(X, y=None, fit_params) Fit to data, then transform it. Fits transformer to `X` and `y` with optional parameters `fit_params` and returns a transformed version of `X`. Parameters X : array-like of shape (n_samples, n_features) Input samples. y : array-like of shape (n_samples,) or (n_samples, n_outputs), default=None Target values (None for unsupervised transformations). **fit_params : dict Additional fit parameters. Returns X_new : ndarray array of shape (n_samples, n_features_new) Transformed array. get_params(deep=True) Return estimator parameter names for GridSearch support. predict(X) Predict target values for X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns y_target : array-like, shape = [n_samples] or [n_samples, n_targets] Predicted target values. predict_meta_features(X) Get meta-features of test-data. Parameters X : numpy array, shape = [n_samples, n_features] Test vectors, where n_samples is the number of samples and n_features is the number of features. Returns meta-features : numpy array, shape = [n_samples, len(self.regressors)] meta-features for test data, where n_samples is the number of samples in test data and len(self.regressors) is the number of regressors. If self.multi_output is True, then the number of columns is len(self.regressors) * n_targets score(X, y, sample_weight=None) Return the coefficient of determination :math: R^2 of the prediction. The coefficient :math:`R^2` is defined as :math:`(1 - \\frac{u}{v})`, where :math:`u` is the residual sum of squares ``((y_true - y_pred) 2).sum() and :math:`v` is the total sum of squares ((y_true - y_true.mean()) 2).sum()``. The best possible score is 1.0 and it can be negative (because the model can be arbitrarily worse). A constant model that always predicts the expected value of y , disregarding the input features, would get a :math: R^2 score of 0.0. Parameters X : array-like of shape (n_samples, n_features) Test samples. For some estimators this may be a precomputed kernel matrix or a list of generic objects instead with shape (n_samples, n_samples_fitted) , where n_samples_fitted is the number of samples used in the fitting for the estimator. y : array-like of shape (n_samples,) or (n_samples, n_outputs) True values for X . sample_weight : array-like of shape (n_samples,), default=None Sample weights. Returns score : float :math: R^2 of self.predict(X) wrt. y . Notes The :math: R^2 score used when calling score on a regressor uses multioutput='uniform_average' from version 0.23 to keep consistent with default value of :func: ~sklearn.metrics.r2_score . This influences the score method of all the multioutput regressors (except for :class: ~sklearn.multioutput.MultiOutputRegressor ). set_params( params) Set the parameters of this estimator. Valid parameter keys can be listed with ``get_params()``. Returns self","title":"Methods"},{"location":"user_guide/regressor/StackingRegressor/#properties","text":"coef_ None intercept_ None named_regressors None ython","title":"Properties"},{"location":"user_guide/text/generalize_names/","text":"generalize_names: convert names into a generalized format A function that converts a name into a general format <last_name><separator><firstname letter(s)> (all lowercase) . from mlxtend.text import generalize_names Overview A function that converts a name into a general format <last_name><separator><firstname letter(s)> (all lowercase) , which is useful if data is collected from different sources and is supposed to be compared or merged based on name identifiers. E.g., if names are stored in a pandas DataFrame column, the apply function can be used to generalize names: df['name'] = df['name'].apply(generalize_names) References - Example 1 - Defaults from mlxtend.text import generalize_names generalize_names('Pozo, Jos\u00e9 \u00c1ngel') 'pozo j' generalize_names('Jos\u00e9 Pozo') 'pozo j' generalize_names('Jos\u00e9 \u00c1ngel Pozo') 'pozo j' Example 2 - Optional Parameters from mlxtend.text import generalize_names generalize_names(\"Eto'o, Samuel\", firstname_output_letters=2) 'etoo sa' generalize_names(\"Eto'o, Samuel\", firstname_output_letters=0) 'etoo' generalize_names(\"Eto'o, Samuel\", output_sep=', ') 'etoo, s' API generalize_names(name, output_sep=' ', firstname_output_letters=1) Generalize a person's first and last name. Returns a person's name in the format <last_name><separator><firstname letter(s)> (all lowercase) Parameters name : str Name of the player output_sep : str (default: ' ') String for separating last name and first name in the output. firstname_output_letters : int Number of letters in the abbreviated first name. Returns gen_name : str The generalized name. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/text/generalize_names/","title":"generalize_names: convert names into a generalized format"},{"location":"user_guide/text/generalize_names/#generalize_names-convert-names-into-a-generalized-format","text":"A function that converts a name into a general format <last_name><separator><firstname letter(s)> (all lowercase) . from mlxtend.text import generalize_names","title":"generalize_names: convert names into a generalized format"},{"location":"user_guide/text/generalize_names/#overview","text":"A function that converts a name into a general format <last_name><separator><firstname letter(s)> (all lowercase) , which is useful if data is collected from different sources and is supposed to be compared or merged based on name identifiers. E.g., if names are stored in a pandas DataFrame column, the apply function can be used to generalize names: df['name'] = df['name'].apply(generalize_names)","title":"Overview"},{"location":"user_guide/text/generalize_names/#references","text":"-","title":"References"},{"location":"user_guide/text/generalize_names/#example-1-defaults","text":"from mlxtend.text import generalize_names generalize_names('Pozo, Jos\u00e9 \u00c1ngel') 'pozo j' generalize_names('Jos\u00e9 Pozo') 'pozo j' generalize_names('Jos\u00e9 \u00c1ngel Pozo') 'pozo j'","title":"Example 1 - Defaults"},{"location":"user_guide/text/generalize_names/#example-2-optional-parameters","text":"from mlxtend.text import generalize_names generalize_names(\"Eto'o, Samuel\", firstname_output_letters=2) 'etoo sa' generalize_names(\"Eto'o, Samuel\", firstname_output_letters=0) 'etoo' generalize_names(\"Eto'o, Samuel\", output_sep=', ') 'etoo, s'","title":"Example 2 - Optional Parameters"},{"location":"user_guide/text/generalize_names/#api","text":"generalize_names(name, output_sep=' ', firstname_output_letters=1) Generalize a person's first and last name. Returns a person's name in the format <last_name><separator><firstname letter(s)> (all lowercase) Parameters name : str Name of the player output_sep : str (default: ' ') String for separating last name and first name in the output. firstname_output_letters : int Number of letters in the abbreviated first name. Returns gen_name : str The generalized name. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/text/generalize_names/","title":"API"},{"location":"user_guide/text/generalize_names_duplcheck/","text":"generalize_names_duplcheck: Generalize names while preventing duplicates among different names A function that converts a name into a general format <last_name><separator><firstname letter(s)> (all lowercase) in a pandas DataFrame while avoiding duplicate entries. from mlxtend.text import generalize_names_duplcheck Overview Note that using mlxtend.text.generalize_names with few firstname_output_letters can result in duplicate entries. E.g., if your dataset contains the names \"Adam Johnson\" and \"Andrew Johnson\", the default setting (i.e., 1 first name letter) will produce the generalized name \"johnson a\" in both cases. One solution is to increase the number of first name letters in the output by setting the parameter firstname_output_letters to a value larger than 1. An alternative solution is to use the generalize_names_duplcheck function if you are working with pandas DataFrames. By default, generalize_names_duplcheck will apply generalize_names to a pandas DataFrame column with the minimum number of first name letters and append as many first name letters as necessary until no duplicates are present in the given DataFrame column. An example dataset column that contains the names References - Example 1 - Defaults Reading in a CSV file that has column Name for which we want to generalize the names: Samuel Eto'o Adam Johnson Andrew Johnson import pandas as pd from io import StringIO simulated_csv = \"name,some_value\\n\"\\ \"Samuel Eto'o,1\\n\"\\ \"Adam Johnson,1\\n\"\\ \"Andrew Johnson,1\\n\" df = pd.read_csv(StringIO(simulated_csv)) df name some_value 0 Samuel Eto'o 1 1 Adam Johnson 1 2 Andrew Johnson 1 Applying generalize_names_duplcheck to generate a new DataFrame with the generalized names without duplicates: from mlxtend.text import generalize_names_duplcheck df_new = generalize_names_duplcheck(df=df, col_name='name') df_new name some_value 0 etoo s 1 1 johnson ad 1 2 johnson an 1 API generalize_names_duplcheck(df, col_name) Generalizes names and removes duplicates. Applies mlxtend.text.generalize_names to a DataFrame with 1 first name letter by default and uses more first name letters if duplicates are detected. Parameters df : pandas.DataFrame DataFrame that contains a column where generalize_names should be applied. col_name : str Name of the DataFrame column where generalize_names function should be applied to. Returns df_new : str New DataFrame object where generalize_names function has been applied without duplicates. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/text/generalize_names_duplcheck/","title":"generalize_names_duplcheck: Generalize names while preventing duplicates among different names"},{"location":"user_guide/text/generalize_names_duplcheck/#generalize_names_duplcheck-generalize-names-while-preventing-duplicates-among-different-names","text":"A function that converts a name into a general format <last_name><separator><firstname letter(s)> (all lowercase) in a pandas DataFrame while avoiding duplicate entries. from mlxtend.text import generalize_names_duplcheck","title":"generalize_names_duplcheck: Generalize names while preventing duplicates among different names"},{"location":"user_guide/text/generalize_names_duplcheck/#overview","text":"Note that using mlxtend.text.generalize_names with few firstname_output_letters can result in duplicate entries. E.g., if your dataset contains the names \"Adam Johnson\" and \"Andrew Johnson\", the default setting (i.e., 1 first name letter) will produce the generalized name \"johnson a\" in both cases. One solution is to increase the number of first name letters in the output by setting the parameter firstname_output_letters to a value larger than 1. An alternative solution is to use the generalize_names_duplcheck function if you are working with pandas DataFrames. By default, generalize_names_duplcheck will apply generalize_names to a pandas DataFrame column with the minimum number of first name letters and append as many first name letters as necessary until no duplicates are present in the given DataFrame column. An example dataset column that contains the names","title":"Overview"},{"location":"user_guide/text/generalize_names_duplcheck/#references","text":"-","title":"References"},{"location":"user_guide/text/generalize_names_duplcheck/#example-1-defaults","text":"Reading in a CSV file that has column Name for which we want to generalize the names: Samuel Eto'o Adam Johnson Andrew Johnson import pandas as pd from io import StringIO simulated_csv = \"name,some_value\\n\"\\ \"Samuel Eto'o,1\\n\"\\ \"Adam Johnson,1\\n\"\\ \"Andrew Johnson,1\\n\" df = pd.read_csv(StringIO(simulated_csv)) df name some_value 0 Samuel Eto'o 1 1 Adam Johnson 1 2 Andrew Johnson 1 Applying generalize_names_duplcheck to generate a new DataFrame with the generalized names without duplicates: from mlxtend.text import generalize_names_duplcheck df_new = generalize_names_duplcheck(df=df, col_name='name') df_new name some_value 0 etoo s 1 1 johnson ad 1 2 johnson an 1","title":"Example 1 - Defaults"},{"location":"user_guide/text/generalize_names_duplcheck/#api","text":"generalize_names_duplcheck(df, col_name) Generalizes names and removes duplicates. Applies mlxtend.text.generalize_names to a DataFrame with 1 first name letter by default and uses more first name letters if duplicates are detected. Parameters df : pandas.DataFrame DataFrame that contains a column where generalize_names should be applied. col_name : str Name of the DataFrame column where generalize_names function should be applied to. Returns df_new : str New DataFrame object where generalize_names function has been applied without duplicates. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/text/generalize_names_duplcheck/","title":"API"},{"location":"user_guide/text/tokenizer/","text":"tokenizer_emoticons: tokenizers for emoticons Different functions to tokenize text. from mlxtend.text import tokenizer_emoticons from mlxtend.text import tokenizer_words_and_emoticons Overview Different functions to tokenize text for natural language processing tasks, for example such as building a bag-of-words model for text classification. References - Example 1 - Extract Emoticons from mlxtend.text import tokenizer_emoticons tokenizer_emoticons('</a>This :) is :( a test :-)!') [':)', ':(', ':-)'] Example 2 - Extract Words and Emoticons from mlxtend.text import tokenizer_words_and_emoticons tokenizer_words_and_emoticons('</a>This :) is :( a test :-)!') ['this', 'is', 'a', 'test', ':)', ':(', ':-)'] API tokenizer_emoticons(text) Return emoticons from text Examples >>> tokenizer_emoticons('</a>This :) is :( a test :-)!') [':)', ':(', ':-)'] For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/text/tokenizer_emoticons/ tokenizer_words_and_emoticons(text) Convert text to lowercase words and emoticons. Examples >>> tokenizer_words_and_emoticons('</a>This :) is :( a test :-)!') ['this', 'is', 'a', 'test', ':)', ':(', ':-)'] For more usage examples, please see https://rasbt.github.io/mlxtend/user_guide/text/tokenizer_words_and_emoticons/","title":"tokenizer_emoticons: tokenizers for emoticons"},{"location":"user_guide/text/tokenizer/#tokenizer_emoticons-tokenizers-for-emoticons","text":"Different functions to tokenize text. from mlxtend.text import tokenizer_emoticons from mlxtend.text import tokenizer_words_and_emoticons","title":"tokenizer_emoticons: tokenizers for emoticons"},{"location":"user_guide/text/tokenizer/#overview","text":"Different functions to tokenize text for natural language processing tasks, for example such as building a bag-of-words model for text classification.","title":"Overview"},{"location":"user_guide/text/tokenizer/#references","text":"-","title":"References"},{"location":"user_guide/text/tokenizer/#example-1-extract-emoticons","text":"from mlxtend.text import tokenizer_emoticons tokenizer_emoticons('</a>This :) is :( a test :-)!') [':)', ':(', ':-)']","title":"Example 1 - Extract Emoticons"},{"location":"user_guide/text/tokenizer/#example-2-extract-words-and-emoticons","text":"from mlxtend.text import tokenizer_words_and_emoticons tokenizer_words_and_emoticons('</a>This :) is :( a test :-)!') ['this', 'is', 'a', 'test', ':)', ':(', ':-)']","title":"Example 2 - Extract Words and Emoticons"},{"location":"user_guide/text/tokenizer/#api","text":"tokenizer_emoticons(text) Return emoticons from text Examples >>> tokenizer_emoticons('</a>This :) is :( a test :-)!') [':)', ':(', ':-)'] For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/text/tokenizer_emoticons/ tokenizer_words_and_emoticons(text) Convert text to lowercase words and emoticons. Examples >>> tokenizer_words_and_emoticons('</a>This :) is :( a test :-)!') ['this', 'is', 'a', 'test', ':)', ':(', ':-)'] For more usage examples, please see https://rasbt.github.io/mlxtend/user_guide/text/tokenizer_words_and_emoticons/","title":"API"},{"location":"user_guide/utils/Counter/","text":"Counter: A simple progress counter A simple progress counter to print the number of iterations and time elapsed in a for-loop execution. from mlxtend.utils import Counter Overview The Counter class implements an object for displaying the number of iterations and time elapsed in a for-loop. Please note that the Counter was implemented for efficiency; thus, the Counter offers only very basic functionality in order to avoid relatively expensive evaluations (of if-else statements). References - Example 1 - Counting the iterations in a for-loop from mlxtend.utils import Counter import time cnt = Counter() for i in range(20): # do some computation time.sleep(0.1) cnt.update() 20 iter | 2 sec Note that the first number displays the current iteration, and the second number shows the time elapsed after initializing the Counter . API Counter(stderr=False, start_newline=True, precision=0, name=None) Class to display the progress of for-loop iterators. Parameters stderr : bool (default: True) Prints output to sys.stderr if True; uses sys.stdout otherwise. start_newline : bool (default: True) Prepends a new line to the counter, which prevents overwriting counters if multiple counters are printed in succession. precision: int (default: 0) Sets the number of decimal places when displaying the time elapsed in seconds. name : string (default: None) Prepends the specified name before the counter to allow distinguishing between multiple counters. Attributes curr_iter : int The current iteration. start_time : float The system's time in seconds when the Counter was initialized. end_time : float The system's time in seconds when the Counter was last updated. Examples >>> cnt = Counter() >>> for i in range(20): ... # do some computation ... time.sleep(0.1) ... cnt.update() 20 iter | 2 sec >>> print('The counter was initialized.' ' %d seconds ago.' % (time.time() - cnt.start_time)) The counter was initialized 2 seconds ago >>> print('The counter was last updated' ' %d seconds ago.' % (time.time() - cnt.end_time)) The counter was last updated 0 seconds ago. For more usage examples, please see https://rasbt.github.io/mlxtend/user_guide/utils/Counter/ Methods update() Print current iteration and time elapsed.","title":"Counter: A simple progress counter"},{"location":"user_guide/utils/Counter/#counter-a-simple-progress-counter","text":"A simple progress counter to print the number of iterations and time elapsed in a for-loop execution. from mlxtend.utils import Counter","title":"Counter: A simple progress counter"},{"location":"user_guide/utils/Counter/#overview","text":"The Counter class implements an object for displaying the number of iterations and time elapsed in a for-loop. Please note that the Counter was implemented for efficiency; thus, the Counter offers only very basic functionality in order to avoid relatively expensive evaluations (of if-else statements).","title":"Overview"},{"location":"user_guide/utils/Counter/#references","text":"-","title":"References"},{"location":"user_guide/utils/Counter/#example-1-counting-the-iterations-in-a-for-loop","text":"from mlxtend.utils import Counter import time cnt = Counter() for i in range(20): # do some computation time.sleep(0.1) cnt.update() 20 iter | 2 sec Note that the first number displays the current iteration, and the second number shows the time elapsed after initializing the Counter .","title":"Example 1 - Counting the iterations in a for-loop"},{"location":"user_guide/utils/Counter/#api","text":"Counter(stderr=False, start_newline=True, precision=0, name=None) Class to display the progress of for-loop iterators. Parameters stderr : bool (default: True) Prints output to sys.stderr if True; uses sys.stdout otherwise. start_newline : bool (default: True) Prepends a new line to the counter, which prevents overwriting counters if multiple counters are printed in succession. precision: int (default: 0) Sets the number of decimal places when displaying the time elapsed in seconds. name : string (default: None) Prepends the specified name before the counter to allow distinguishing between multiple counters. Attributes curr_iter : int The current iteration. start_time : float The system's time in seconds when the Counter was initialized. end_time : float The system's time in seconds when the Counter was last updated. Examples >>> cnt = Counter() >>> for i in range(20): ... # do some computation ... time.sleep(0.1) ... cnt.update() 20 iter | 2 sec >>> print('The counter was initialized.' ' %d seconds ago.' % (time.time() - cnt.start_time)) The counter was initialized 2 seconds ago >>> print('The counter was last updated' ' %d seconds ago.' % (time.time() - cnt.end_time)) The counter was last updated 0 seconds ago. For more usage examples, please see https://rasbt.github.io/mlxtend/user_guide/utils/Counter/","title":"API"},{"location":"user_guide/utils/Counter/#methods","text":"update() Print current iteration and time elapsed.","title":"Methods"}]}
\ No newline at end of file
+{"config":{"indexing":"full","lang":["en"],"min_search_length":3,"prebuild_index":false,"separator":"[\\s\\-]+"},"docs":[{"location":"","text":"Welcome to mlxtend's documentation! Mlxtend (machine learning extensions) is a Python library of useful tools for the day-to-day data science tasks. Links Documentation: https://rasbt.github.io/mlxtend Source code repository: https://github.com/rasbt/mlxtend PyPI: https://pypi.python.org/pypi/mlxtend Questions? Check out the GitHub Discussions board Examples import numpy as np import matplotlib.pyplot as plt import matplotlib.gridspec as gridspec import itertools from sklearn.linear_model import LogisticRegression from sklearn.svm import SVC from sklearn.ensemble import RandomForestClassifier from mlxtend.classifier import EnsembleVoteClassifier from mlxtend.data import iris_data from mlxtend.plotting import plot_decision_regions # Initializing Classifiers clf1 = LogisticRegression(random_state=0) clf2 = RandomForestClassifier(random_state=0) clf3 = SVC(random_state=0, probability=True) eclf = EnsembleVoteClassifier(clfs=[clf1, clf2, clf3], weights=[2, 1, 1], voting='soft') # Loading some example data X, y = iris_data() X = X[:,[0, 2]] # Plotting Decision Regions gs = gridspec.GridSpec(2, 2) fig = plt.figure(figsize=(10, 8)) labels = ['Logistic Regression', 'Random Forest', 'RBF kernel SVM', 'Ensemble'] for clf, lab, grd in zip([clf1, clf2, clf3, eclf], labels, itertools.product([0, 1], repeat=2)): clf.fit(X, y) ax = plt.subplot(gs[grd[0], grd[1]]) fig = plot_decision_regions(X=X, y=y, clf=clf, legend=2) plt.title(lab) plt.show() If you use mlxtend as part of your workflow in a scientific publication, please consider citing the mlxtend repository with the following DOI: @article{raschkas_2018_mlxtend, author = {Sebastian Raschka}, title = {MLxtend: Providing machine learning and data science utilities and extensions to Python\u2019s scientific computing stack}, journal = {The Journal of Open Source Software}, volume = {3}, number = {24}, month = apr, year = 2018, publisher = {The Open Journal}, doi = {10.21105/joss.00638}, url = {https://joss.theoj.org/papers/10.21105/joss.00638} } License This project is released under a permissive new BSD open source license ( LICENSE-BSD3.txt ) and commercially usable. There is no warranty; not even for merchantability or fitness for a particular purpose. In addition, you may use, copy, modify and redistribute all artistic creative works (figures and images) included in this distribution under the directory according to the terms and conditions of the Creative Commons Attribution 4.0 International License. See the file LICENSE-CC-BY.txt for details. (Computer-generated graphics such as the plots produced by matplotlib fall under the BSD license mentioned above). Contact I received a lot of feedback and questions about mlxtend recently, and I thought that it would be worthwhile to set up a public communication channel. Before you write an email with a question about mlxtend, please consider posting it here since it can also be useful to others! Please join the Google Groups Mailing List ! If Google Groups is not for you, please feel free to write me an email or consider filing an issue on GitHub's issue tracker for new feature requests or bug reports. In addition, I setup a Gitter channel for live discussions.","title":"Home"},{"location":"#welcome-to-mlxtends-documentation","text":"Mlxtend (machine learning extensions) is a Python library of useful tools for the day-to-day data science tasks.","title":"Welcome to mlxtend's documentation!"},{"location":"#links","text":"Documentation: https://rasbt.github.io/mlxtend Source code repository: https://github.com/rasbt/mlxtend PyPI: https://pypi.python.org/pypi/mlxtend Questions? Check out the GitHub Discussions board","title":"Links"},{"location":"#examples","text":"import numpy as np import matplotlib.pyplot as plt import matplotlib.gridspec as gridspec import itertools from sklearn.linear_model import LogisticRegression from sklearn.svm import SVC from sklearn.ensemble import RandomForestClassifier from mlxtend.classifier import EnsembleVoteClassifier from mlxtend.data import iris_data from mlxtend.plotting import plot_decision_regions # Initializing Classifiers clf1 = LogisticRegression(random_state=0) clf2 = RandomForestClassifier(random_state=0) clf3 = SVC(random_state=0, probability=True) eclf = EnsembleVoteClassifier(clfs=[clf1, clf2, clf3], weights=[2, 1, 1], voting='soft') # Loading some example data X, y = iris_data() X = X[:,[0, 2]] # Plotting Decision Regions gs = gridspec.GridSpec(2, 2) fig = plt.figure(figsize=(10, 8)) labels = ['Logistic Regression', 'Random Forest', 'RBF kernel SVM', 'Ensemble'] for clf, lab, grd in zip([clf1, clf2, clf3, eclf], labels, itertools.product([0, 1], repeat=2)): clf.fit(X, y) ax = plt.subplot(gs[grd[0], grd[1]]) fig = plot_decision_regions(X=X, y=y, clf=clf, legend=2) plt.title(lab) plt.show() If you use mlxtend as part of your workflow in a scientific publication, please consider citing the mlxtend repository with the following DOI: @article{raschkas_2018_mlxtend, author = {Sebastian Raschka}, title = {MLxtend: Providing machine learning and data science utilities and extensions to Python\u2019s scientific computing stack}, journal = {The Journal of Open Source Software}, volume = {3}, number = {24}, month = apr, year = 2018, publisher = {The Open Journal}, doi = {10.21105/joss.00638}, url = {https://joss.theoj.org/papers/10.21105/joss.00638} }","title":"Examples"},{"location":"#license","text":"This project is released under a permissive new BSD open source license ( LICENSE-BSD3.txt ) and commercially usable. There is no warranty; not even for merchantability or fitness for a particular purpose. In addition, you may use, copy, modify and redistribute all artistic creative works (figures and images) included in this distribution under the directory according to the terms and conditions of the Creative Commons Attribution 4.0 International License. See the file LICENSE-CC-BY.txt for details. (Computer-generated graphics such as the plots produced by matplotlib fall under the BSD license mentioned above).","title":"License"},{"location":"#contact","text":"I received a lot of feedback and questions about mlxtend recently, and I thought that it would be worthwhile to set up a public communication channel. Before you write an email with a question about mlxtend, please consider posting it here since it can also be useful to others! Please join the Google Groups Mailing List ! If Google Groups is not for you, please feel free to write me an email or consider filing an issue on GitHub's issue tracker for new feature requests or bug reports. In addition, I setup a Gitter channel for live discussions.","title":"Contact"},{"location":"CHANGELOG/","text":"Release Notes The CHANGELOG for the current development version is available at https://github.com/rasbt/mlxtend/blob/master/docs/sources/CHANGELOG.md . Version 0.23.3 (15 Nov 2024) Downloads Source code (zip) Source code (tar.gz) New Features and Enhancements Files updated: - mlxtend.evaluate.time_series.plot_splits - Improved plot_splits for better visualization of time series splits Changes mlxtend/feature_selection/exhaustive_feature_selector.py np.inf update to support for NumPy 2.0 Version 0.23.2 (5 Nov 2024) Downloads Source code (zip) Source code (tar.gz) New Features and Enhancements Implement the FP-Growth and FP-Max algorithms with the possibility of missing values in the input dataset. Added a new metric Representativity for the association rules generated ( #1004 via zazass8 ). Files updated: ['mlxtend.frequent_patterns.fpcommon'] 'mlxtend.frequent_patterns.fpgrowth' 'mlxtend.frequent_patterns.fpmax' 'mlxtend/feature_selection/utilities.py' Modified _calc_score function to ensure compatibility with scikit-learn versions 1.4 and above by dynamically selecting between fit_params and params in cross_val_score . mlxtend.feature_selection.SequentialFeatureSelector Updated negative infinity constant to be compatible with old and new (>=2.0) numpy versions mlxtend.frequent_patterns.association_rules Implemented three new metrics: Jaccard, Certainty, and Kulczynski. ( #1096 ) Integrated scikit-learn's set_output method into TransactionEncoder ( #1087 via it176131 ) Changes [ mlxtend.frequent_patterns.fpcommon ] Added the null_values parameter in valid_input_check signature to check in case the input also includes null values. Changes the returns statements and function signatures for setup_fptree and generated_itemsets respectively to return the disabled array created and to include it as a parameter. Added code in [ mlxtend.frequent_patterns.fpcommon ] and mlxtend.frequent_patterns.association_rules to implement the algorithms in case null values exist when null_values is True. mlxtend.frequent_patterns.association_rules Added optional parameter 'return_metrics' to only return a given list of metrics, rather than every possible metric. Add n_classes_ attribute to stacking classifiers for compatibility with scikit-learn 1.3 ( #1091 ) Use Scipy's instead of NumPy's decompositions in PCA for improved accuracy in edge cases ( #1080 via [ fkdosilovic ]) Version 0.23.1 (5 Jan 2024) Downloads Source code (zip) Source code (tar.gz) Changes Updated dependency on distutils for python 3.12 and above ( #1072 via peanutsee ) Version 0.23.0 (22 Sep 2023) Downloads Source code (zip) Source code (tar.gz) Changes Address NumPy deprecations to make mlxtend compatible to NumPy 1.24 Changed the signature of the LinearRegression model of sklearn in the test removing the normalize parameter as it is deprecated. ( #1036 ) Add pyproject.toml to support PEP 518 builds ( #1065 via jmahlik ) Fixed installation from sdist failing ( #1065 via jmahlik ) Converted configuration to pyproject.toml ( #1065 via jmahlik ) Remove mlxtend.image submodule with face recognition functions due to poor dlib support in modern environments. New Features and Enhancements Document how to use SequentialFeatureSelector and multiclass ROC AUC. Version 0.22.0 (4 April 2023) Downloads Source code (zip) Source code (tar.gz) Changes When ExhaustiveFeatureSelector is run with n_jobs == 1 , joblib is now disabled, which enables more immediate (live) feedback when the verbose mode is enabled. ( #985 via Nima Sarajpoor ) Disabled unnecessary warning in EnsembleVoteClassifier ( #941 ) Fixed various documentation issues ( #849 and #951 via Lekshmanan Natarajan ) Fixed \"Edit on GitHub\" button ( #1024 ) New Features and Enhancements The mlxtend.frequent_patterns.association_rules function has a new metric - Zhang's Metric, which measures both association and dissociation. ( #980 ) Internal mlxtend.frequent_patterns.fpmax code improvement that avoids casting a sparse DataFrame into a dense NumPy array. ( #1000 via Tim Kellogg ) The plot_decision_regions function now has a n_jobs parameter to parallelize the computation. (In a particular use case, on a small dataset, there was a 21x speed-up (449 seconds vs 21 seconds on local HPC instance of 36 cores). ( #998 via Khalid ElHaj ) Added mlxtend.frequent_patterns.hmine algorithm and documentation for mining frequent itemsets using the H-Mine algorithm. ( #1020 via Fatih Sen ) Version 0.21.0 (09/17/2022) Downloads Source code (zip) Source code (tar.gz) New Features and Enhancements The mlxtend.evaluate.feature_importance_permutation function has a new feature_groups argument to treat user-specified feature groups as single features, which is useful for one-hot encoded features. ( #955 ) The mlxtend.feature_selection.ExhaustiveFeatureSelector and SequentialFeatureSelector also gained support for feature_groups with a behavior similar to the one described above. ( #957 and #965 via Nima Sarajpoor ) Changes The custom_feature_names parameter was removed from the ExhaustiveFeatureSelector due to redundancy and to simplify the code base. The ExhaustiveFeatureSelector documentation illustrates how the same behavior and outcome can be achieved using pandas DataFrames. ( #957 ) Bug Fixes None Version 0.20.0 (05/26/2022) Downloads Source code (zip) Source code (tar.gz) New Features and Enhancements The mlxtend.evaluate.bootstrap_point632_score now supports fit_params . ( #861 ) The mlxtend/plotting/decision_regions.py function now has a contourf_kwargs for matplotlib to change the look of the decision boundaries if desired. ( #881 via [ pbloem ]) Add a norm_colormap parameter to mlxtend.plotting.plot_confusion_matrix , to allow normalizing the colormap, e.g., using matplotlib.colors.LogNorm() ( #895 ) Add new GroupTimeSeriesSplit class for evaluation in time series tasks with support of custom groups and additional parameters in comparison with scikit-learn's TimeSeriesSplit . ( #915 via Dmitry Labazkin ) Changes Due to compatibility issues with newer package versions, certain functions from six.py have been removed so that mlxtend may not work anymore with Python 2.7. As an internal change to speed up unit testing, unit testing is now faciliated by GitHub workflows, and Travis CI and Appveyor hooks have been removed. Improved axis label rotation in mlxtend.plotting.heatmap and mlxtend.plotting.plot_confusion_matrix ( #872 ) Fix various typos in McNemar guides. Raises a warning if non-bool arrays are used in the frequent pattern functions apriori , fpmax , and fpgrowth . ( #934 via NimaSarajpoor ) Bug Fixes Fix unreadable labels in heatmap for certain colormaps. ( #852 ) Fix an issue in mlxtend.plotting.plot_confusion_matrix when string class names are passed ( #894 ) Version 0.19.0 (2021-09-02) Downloads Source code (zip) Source code (tar.gz) New Features Adds a second \"balanced accuracy\" interpretation (\"balanced\") to evaluate.accuracy_score in addition to the existing \"average\" option to compute the scikit-learn-style balanced accuracy. ( #764 ) Adds new scatter_hist function to mlxtend.plotting for generating a scattered histogram. ( #757 via Maitreyee Mhasaka ) The evaluate.permutation_test function now accepts a paired argument to specify to support paired permutation/randomization tests. ( #768 ) The StackingCVRegressor now also supports multi-dimensional targets similar to StackingRegressor via StackingCVRegressor(..., multi_output=True) . ( #802 via Marco Tiraboschi ) Changes Updates unit tests for scikit-learn 0.24.1 compatibility. ( #774 ) StackingRegressor now requires setting StackingRegressor(..., multi_output=True) if the target is multi-dimensional; this allows for better input validation. ( #802 ) Removes deprecated res argument from plot_decision_regions . ( #803 ) Adds a title_fontsize parameter to plot_learning_curves for controlling the title font size; also the plot style is now the matplotlib default. ( #818 ) Internal change using 'c': 'none' instead of 'c': '' in mlxtend.plotting.plot_decision_regions 's scatterplot highlights to stay compatible with Matplotlib 3.4 and newer. ( #822 ) Adds a fontcolor_threshold parameter to the mlxtend.plotting.plot_confusion_matrix function as an additional option for determining the font color cut-off manually. ( #827 ) The frequent_patterns.association_rules now raises a ValueError if an empty frequent itemset DataFrame is passed. ( #843 ) The .632 and .632+ bootstrap method implemented in the mlxtend.evaluate.bootstrap_point632_score function now use the whole training set for the resubstitution weighting term instead of the internal training set that is a new bootstrap sample in each round. ( #844 ) Bug Fixes Fixes a typo in the SequentialFeatureSelector documentation ( #835 via Jo\u00e3o Pedro Zanlorensi Cardoso ) Version 0.18.0 (2020-11-25) Downloads Source code (zip) Source code (tar.gz) New Features The bias_variance_decomp function now supports optional fit_params for the estimators that are fit on bootstrap samples. ( #748 ) The bias_variance_decomp function now supports Keras estimators. ( #725 via @hanzigs ) Adds new mlxtend.classifier.OneRClassifier (One Rule Classfier) class, a simple rule-based classifier that is often used as a performance baseline or simple interpretable model. ( #726 Adds new create_counterfactual method for creating counterfactuals to explain model predictions. ( #740 ) Changes permutation_test ( mlxtend.evaluate.permutation ) \u00ecs corrected to give the proportion of permutations whose statistic is at least as extreme as the one observed. ( #721 via Florian Charlier ) Fixes the McNemar confusion matrix layout to match the convention (and documentation), swapping the upper left and lower right cells. ( #744 via mmarius ) Bug Fixes The loss in LogisticRegression for logging purposes didn't include the L2 penalty for the first weight in the weight vector (this is not the bias unit). However, since this loss function was only used for logging purposes, and the gradient remains correct, this does not have an effect on the main code. ( #741 ) Fixes a bug in bias_variance_decomp where when the mse loss was used, downcasting to integers caused imprecise results for small numbers. ( #749 ) Version 0.17.3 (2020-07-27) Downloads Source code (zip) Source code (tar.gz) New Features Add predict_proba kwarg to bootstrap methods, to allow bootstrapping of scoring functions that take in probability values. ( #700 via Adam Li ) Add a cell_values parameter to mlxtend.plotting.heatmap() to optionally suppress cell annotations by setting cell_values=False . ( #703 Changes Implemented both use_clones and fit_base_estimators (previously refit in EnsembleVoteClassifier ) for EnsembleVoteClassifier and StackingClassifier . ( #670 via Katrina Ni ) Switched to using raw strings for regex in mlxtend.text to prevent deprecation warning in Python 3.8 ( #688 ) Slice data in sequential forward selection before sending to parallel backend, reducing memory consumption. Bug Fixes Fixes axis DeprecationWarning in matplotlib v3.1.0 and newer. ( #673 ) Fixes an issue with using meshgrid in no_information_rate function used by the bootstrap_point632_score function for the .632+ estimate. ( #688 ) Fixes an issue in fpmax that could lead to incorrect support values. ( #692 via Steve Harenberg ) Version 0.17.2 (2020-02-24) Downloads Source code (zip) Source code (tar.gz) New Features - Changes The previously deprecated OnehotTransactions has been removed in favor of the TransactionEncoder. Removed SparseDataFrame support in frequent pattern mining functions in favor of pandas >=1.0's new way for working sparse data. If you used SparseDataFrame formats, please see pandas' migration guide at https://pandas.pydata.org/pandas-docs/stable/user_guide/sparse.html#migrating. ( #667 ) The plot_confusion_matrix.py now also accepts a matplotlib figure and axis as input to which the confusion matrix plot can be added. ( #671 via Vahid Mirjalili ) Bug Fixes - Version 0.17.1 (2020-01-28) Downloads Source code (zip) Source code (tar.gz) New Features The SequentialFeatureSelector now supports using pre-specified feature sets via the fixed_features parameter. ( #578 ) Adds a new accuracy_score function to mlxtend.evaluate for computing basic classifcation accuracy, per-class accuracy, and average per-class accuracy. ( #624 via Deepan Das ) StackingClassifier and StackingCVClassifier now have a decision_function method, which serves as a preferred choice over predict_proba in calculating roc_auc and average_precision scores when the meta estimator is a linear model or support vector classifier. ( #634 via Qiang Gu ) Changes Improve the runtime performance for the apriori frequent itemset generating function when low_memory=True . Setting low_memory=False (default) is still faster for small itemsets, but low_memory=True can be much faster for large itemsets and requires less memory. Also, input validation for apriori , \u0300 fpgrowth and fpmax takes a significant amount of time when input pandas DataFrame is large; this is now dramatically reduced when input contains boolean values (and not zeros/ones), which is the case when using TransactionEncoder`. ( #619 via Denis Barbier ) Add support for newer sparse pandas DataFrame for frequent itemset algorithms. Also, input validation for apriori , \u0300 fpgrowth and fpmax` runs much faster on sparse DataFrame when input pandas DataFrame contains integer values. ( #621 via Denis Barbier ) Let fpgrowth and fpmax directly work on sparse DataFrame, they were previously converted into dense Numpy arrays. ( #622 via Denis Barbier ) Bug Fixes Fixes a bug in mlxtend.plotting.plot_pca_correlation_graph that caused the explaind variances not summing up to 1. Also, improves the runtime performance of the correlation computation and adds a missing function argument for the explained variances (eigenvalues) if users provide their own principal components. ( #593 via Gabriel Azevedo Ferreira ) Behavior of fpgrowth and apriori consistent for edgecases such as min_support=0 . ( #573 via Steve Harenberg ) fpmax returns an empty data frame now instead of raising an error if the frequent itemset set is empty. ( #573 via Steve Harenberg ) Fixes and issue in mlxtend.plotting.plot_confusion_matrix , where the font-color choice for medium-dark cells was not ideal and hard to read. #588 via sohrabtowfighi ) The svd mode of mlxtend.feature_extraction.PrincipalComponentAnalysis now also n-1 degrees of freedom instead of n d.o.f. when computing the eigenvalues to match the behavior of eigen . #595 Disable input validation for StackingCVClassifier because it causes issues if pipelines are used as input. #606 Version 0.17.0 (2019-07-19) Downloads Source code (zip) Source code (tar.gz) New Features Added an enhancement to the existing iris_data() such that both the UCI Repository version of the Iris dataset as well as the corrected, original version of the dataset can be loaded, which has a slight difference in two data points (consistent with Fisher's paper; this is also the same as in R). (via #539 via janismdhanbad ) Added optional groups parameter to SequentialFeatureSelector and ExhaustiveFeatureSelector fit() methods for forwarding to sklearn CV ( #537 via arc12 ) Added a new plot_pca_correlation_graph function to the mlxtend.plotting submodule for plotting a PCA correlation graph. ( #544 via Gabriel-Azevedo-Ferreira ) Added a zoom_factor parameter to the mlxten.plotting.plot_decision_region function that allows users to zoom in and out of the decision region plots. ( #545 ) Added a function fpgrowth that implements the FP-Growth algorithm for mining frequent itemsets as a drop-in replacement for the existing apriori algorithm. ( #550 via Steve Harenberg ) New heatmap function in mlxtend.plotting . ( #552 ) Added a function fpmax that implements the FP-Max algorithm for mining maximal itemsets as a drop-in replacement for the fpgrowth algorithm. ( #553 via Steve Harenberg ) New figsize parameter for the plot_decision_regions function in mlxtend.plotting . ( #555 via Mirza Hasanbasic ) New low_memory option for the apriori frequent itemset generating function. Setting low_memory=False (default) uses a substantially optimized version of the algorithm that is 3-6x faster than the original implementation ( low_memory=True ). ( #567 via jmayse ) Added numerically stable OLS methods which uses QR decomposition and Singular Value Decomposition (SVD) methods to LinearRegression in mlxtend.regressor.linear_regression . ( #575 via PuneetGrov3r ) Changes Now uses the latest joblib library under the hood for multiprocessing instead of sklearn.externals.joblib . ( #547 ) Changes to StackingCVClassifier and StackingCVRegressor such that first-level models are allowed to generate output of non-numeric type. ( #562 ) Bug Fixes Fixed documentation of iris_data() under iris.py by adding a note about differences in the iris data in R and UCI machine learning repo. Make sure that if the 'svd' mode is used in PCA, the number of eigenvalues is the same as when using 'eigen' (append 0's zeros in that case) ( #565 ) Version 0.16.0 (2019-05-12) Downloads Source code (zip) Source code (tar.gz) New Features StackingCVClassifier and StackingCVRegressor now support random_state parameter, which, together with shuffle , controls the randomness in the cv splitting. ( #523 via Qiang Gu ) StackingCVClassifier and StackingCVRegressor now have a new drop_last_proba parameter. It drops the last \"probability\" column in the feature set since if True , because it is redundant: p(y_c) = 1 - p(y_1) + p(y_2) + ... + p(y_{c-1}). This can be useful for meta-classifiers that are sensitive to perfectly collinear features. ( #532 ) Other stacking estimators, including StackingClassifier , StackingCVClassifier and StackingRegressor , support grid search over the regressors and even a single base regressor. ( #522 via Qiang Gu ) Adds multiprocessing support to StackingCVClassifier . ( #522 via Qiang Gu ) Adds multiprocessing support to StackingCVRegressor . ( #512 via Qiang Gu ) Now, the StackingCVRegressor also enables grid search over the regressors and even a single base regressor. When there are level-mixed parameters, GridSearchCV will try to replace hyperparameters in a top-down order (see the documentation for examples details). ( #515 via Qiang Gu ) Adds a verbose parameter to apriori to show the current iteration number as well as the itemset size currently being sampled. ( #519 Adds an optional class_name parameter to the confusion matrix function to display class names on the axis as tick marks. ( #487 via sandpiturtle ) Adds a pca.e_vals_normalized_ attribute to PCA for storing the eigenvalues also in normalized form; this is commonly referred to as variance explained ratios. #545 Changes Due to new features, restructuring, and better scikit-learn support (for GridSearchCV , etc.) the StackingCVRegressor 's meta regressor is now being accessed via 'meta_regressor__* in the parameter grid. E.g., if a RandomForestRegressor as meta- egressor was previously tuned via 'randomforestregressor__n_estimators' , this has now changed to 'meta_regressor__n_estimators' . ( #515 via Qiang Gu ) The same change mentioned above is now applied to other stacking estimators, including StackingClassifier , StackingCVClassifier and StackingRegressor . ( #522 via Qiang Gu ) Automatically performs mean centering for PCA solver 'SVD' such that using SVD is always equal to using the covariance matrix approach #545 Bug Fixes The feature_selection.ColumnSelector now also supports column names of type int (in addition to str names) if the input is a pandas DataFrame. ( #500 via tetrar124 Fix unreadable labels in plot_confusion_matrix for imbalanced datasets if show_absolute=True and show_normed=True . ( #504 ) Raises a more informative error if a SparseDataFrame is passed to apriori and the dataframe has integer column names that don't start with 0 due to current limitations of the SparseDataFrame implementation in pandas. ( #503 ) SequentialFeatureSelector now supports DataFrame as input for all operating modes (forward/backward/floating). #506 mlxtend.evaluate.feature_importance_permutation now correctly accepts scoring functions with proper function signature as metric argument. #528 Version 0.15.0 (2019-01-19) Downloads Source code (zip) Source code (tar.gz) New Features Adds a new transformer class to mlxtend.image , EyepadAlign , that aligns face images based on the location of the eyes. ( #466 by Vahid Mirjalili ) Adds a new function, mlxtend.evaluate.bias_variance_decomp that decomposes the loss of a regressor or classifier into bias and variance terms. ( #470 ) Adds a whitening parameter to PrincipalComponentAnalysis , to optionally whiten the transformed data such that the features have unit variance. ( #475 ) Changes Changed the default solver in PrincipalComponentAnalysis to 'svd' instead of 'eigen' to improve numerical stability. ( #474 ) The mlxtend.image.extract_face_landmarks now returns None if no facial landmarks were detected instead of an array of all zeros. ( #466 ) Bug Fixes The eigenvectors maybe have not been sorted in certain edge cases if solver was 'eigen' in PrincipalComponentAnalysis and LinearDiscriminantAnalysis . ( #477 , #478 ) Version 0.14.0 (2018-11-09) Downloads Source code (zip) Source code (tar.gz) New Features Added a scatterplotmatrix function to the plotting module. ( #437 ) Added sample_weight option to StackingRegressor , StackingClassifier , StackingCVRegressor , StackingCVClassifier , EnsembleVoteClassifier . ( #438 ) Added a RandomHoldoutSplit class to perform a random train/valid split without rotation in SequentialFeatureSelector , scikit-learn GridSearchCV etc. ( #442 ) Added a PredefinedHoldoutSplit class to perform a train/valid split, based on user-specified indices, without rotation in SequentialFeatureSelector , scikit-learn GridSearchCV etc. ( #443 ) Created a new mlxtend.image submodule for working on image processing-related tasks. ( #457 ) Added a new convenience function extract_face_landmarks based on dlib to mlxtend.image . ( #458 ) Added a method='oob' option to the mlxtend.evaluate.bootstrap_point632_score method to compute the classic out-of-bag bootstrap estimate ( #459 ) Added a method='.632+' option to the mlxtend.evaluate.bootstrap_point632_score method to compute the .632+ bootstrap estimate that addresses the optimism bias of the .632 bootstrap ( #459 ) Added a new mlxtend.evaluate.ftest function to perform an F-test for comparing the accuracies of two or more classification models. ( #460 ) Added a new mlxtend.evaluate.combined_ftest_5x2cv function to perform an combined 5x2cv F-Test for comparing the performance of two models. ( #461 ) Added a new mlxtend.evaluate.difference_proportions test for comparing two proportions (e.g., classifier accuracies) ( #462 ) Changes Addressed deprecations warnings in NumPy 0.15. ( #425 ) Because of complications in PR ( #459 ), Python 2.7 was now dropped; since official support for Python 2.7 by the Python Software Foundation is ending in approx. 12 months anyways, this re-focussing will hopefully free up some developer time with regard to not having to worry about backward compatibility Bug Fixes Fixed an issue with a missing import in mlxtend.plotting.plot_confusion_matrix . ( #428 ) Version 0.13.0 (2018-07-20) Downloads Source code (zip) Source code (tar.gz) New Features A meaningful error message is now raised when a cross-validation generator is used with SequentialFeatureSelector . ( #377 ) The SequentialFeatureSelector now accepts custom feature names via the fit method for more interpretable feature subset reports. ( #379 ) The SequentialFeatureSelector is now also compatible with Pandas DataFrames and uses DataFrame column-names for more interpretable feature subset reports. ( #379 ) ColumnSelector now works with Pandas DataFrames columns. ( #378 by Manuel Garrido ) The ExhaustiveFeatureSelector estimator in mlxtend.feature_selection now is safely stoppable mid-process by control+c. ( #380 ) Two new functions, vectorspace_orthonormalization and vectorspace_dimensionality were added to mlxtend.math to use the Gram-Schmidt process to convert a set of linearly independent vectors into a set of orthonormal basis vectors, and to compute the dimensionality of a vectorspace, respectively. ( #382 ) mlxtend.frequent_patterns.apriori now supports pandas SparseDataFrame s to generate frequent itemsets. ( #404 via Daniel Morales ) The plot_confusion_matrix function now has the ability to show normalized confusion matrix coefficients in addition to or instead of absolute confusion matrix coefficients with or without a colorbar. The text display method has been changed so that the full range of the colormap is used. The default size is also now set based on the number of classes. Added support for merging the meta features with the original input features in StackingRegressor (via use_features_in_secondary ) like it is already supported in the other Stacking classes. ( #418 ) Added a support_only to the association_rules function, which allow constructing association rules (based on the support metric only) for cropped input DataFrames that don't contain a complete set of antecedent and consequent support values. ( #421 ) Changes Itemsets generated with apriori are now frozenset s ( #393 by William Laney and #394 ) Now raises an error if a input DataFrame to apriori contains non 0, 1, True, False values. #419 ) Bug Fixes Allow mlxtend estimators to be cloned via scikit-learn's clone function. ( #374 ) Fixes bug to allow the correct use of refit=False in StackingRegressor and StackingCVRegressor ( #384 and ( #385 ) by selay01 ) Allow StackingClassifier to work with sparse matrices when use_features_in_secondary=True ( #408 by Floris Hoogenbook ) Allow StackingCVRegressor to work with sparse matrices when use_features_in_secondary=True ( #416 ) Allow StackingCVClassifier to work with sparse matrices when use_features_in_secondary=True ( #417 ) Version 0.12.0 (2018-21-04) Downloads Source code (zip) Source code (tar.gz) New Features A new feature_importance_permuation function to compute the feature importance in classifiers and regressors via the permutation importance method ( #358 ) The fit method of the ExhaustiveFeatureSelector now optionally accepts **fit_params for the estimator that is used for the feature selection. ( #354 by Zach Griffith) The fit method of the SequentialFeatureSelector now optionally accepts **fit_params for the estimator that is used for the feature selection. ( #350 by Zach Griffith) Changes Replaced plot_decision_regions colors by a colorblind-friendly palette and adds contour lines for decision regions. ( #348 ) All stacking estimators now raise NonFittedErrors if any method for inference is called prior to fitting the estimator. ( #353 ) Renamed the refit parameter of both the StackingClassifier and StackingCVClassifier to use_clones to be more explicit and less misleading. ( #368 ) Bug Fixes Various changes in the documentation and documentation tools to fix formatting issues ( #363 ) Fixes a bug where the StackingCVClassifier 's meta features were not stored in the original order when shuffle=True ( #370 ) Many documentation improvements, including links to the User Guides in the API docs ( #371 ) Version 0.11.0 (2018-03-14) Downloads Source code (zip) Source code (tar.gz) New Features New function implementing the resampled paired t-test procedure ( paired_ttest_resampled ) to compare the performance of two models. ( #323 ) New function implementing the k-fold paired t-test procedure ( paired_ttest_kfold_cv ) to compare the performance of two models (also called k-hold-out paired t-test). ( #324 ) New function implementing the 5x2cv paired t-test procedure ( paired_ttest_5x2cv ) proposed by Dieterrich (1998) to compare the performance of two models. ( #325 ) A refit parameter was added to stacking classes (similar to the refit parameter in the EnsembleVoteClassifier ), to support classifiers and regressors that follow the scikit-learn API but are not compatible with scikit-learn's clone function. ( #322 ) The ColumnSelector now has a drop_axis argument to use it in pipelines with CountVectorizers . ( #333 ) Changes Raises an informative error message if predict or predict_meta_features is called prior to calling the fit method in StackingRegressor and StackingCVRegressor . ( #315 ) The plot_decision_regions function now automatically determines the optimal setting based on the feature dimensions and supports anti-aliasing. The old res parameter has been deprecated. ( #309 by Guillaume Poirier-Morency ) Apriori code is faster due to optimization in onehot transformation and the amount of candidates generated by the apriori algorithm. ( #327 by Jakub Smid ) The OnehotTransactions class (which is typically often used in combination with the apriori function for association rule mining) is now more memory efficient as it uses boolean arrays instead of integer arrays. In addition, the OnehotTransactions class can be now be provided with sparse argument to generate sparse representations of the onehot matrix to further improve memory efficiency. ( #328 by Jakub Smid ) The OneHotTransactions has been deprecated and replaced by the TransactionEncoder . ( #332 The plot_decision_regions function now has three new parameters, scatter_kwargs , contourf_kwargs , and scatter_highlight_kwargs , that can be used to modify the plotting style. ( #342 by James Bourbeau ) Bug Fixes Fixed issue when class labels were provided to the EnsembleVoteClassifier when refit was set to false . ( #322 ) Allow arrays with 16-bit and 32-bit precision in plot_decision_regions function. ( #337 ) Fixed bug that raised an indexing error if the number of items was <= 1 when computing association rules using the conviction metric. ( #340 ) Version 0.10.0 (2017-12-22) Downloads Source code (zip) Source code (tar.gz) New Features New store_train_meta_features parameter for fit in StackingCVRegressor. if True, train meta-features are stored in self.train_meta_features_ . New pred_meta_features method for StackingCVRegressor . People can get test meta-features using this method. ( #294 via takashioya ) The new store_train_meta_features attribute and pred_meta_features method for the StackingCVRegressor were also added to the StackingRegressor , StackingClassifier , and StackingCVClassifier ( #299 & #300 ) New function ( evaluate.mcnemar_tables ) for creating multiple 2x2 contigency from model predictions arrays that can be used in multiple McNemar (post-hoc) tests or Cochran's Q or F tests, etc. ( #307 ) New function ( evaluate.cochrans_q ) for performing Cochran's Q test to compare the accuracy of multiple classifiers. ( #310 ) Changes Added requirements.txt to setup.py . ( #304 via Colin Carrol ) Bug Fixes Improved numerical stability for p-values computed via the the exact McNemar test ( #306 ) nose is not required to use the library ( #302 ) Version 0.9.1 (2017-11-19) Downloads Source code (zip) Source code (tar.gz) New Features Added mlxtend.evaluate.bootstrap_point632_score to evaluate the performance of estimators using the .632 bootstrap. ( #283 ) New max_len parameter for the frequent itemset generation via the apriori function to allow for early stopping. ( #270 ) Changes All feature index tuples in SequentialFeatureSelector or now in sorted order. ( #262 ) The SequentialFeatureSelector now runs the continuation of the floating inclusion/exclusion as described in Novovicova & Kittler (1994). Note that this didn't cause any difference in performance on any of the test scenarios but could lead to better performance in certain edge cases. ( #262 ) utils.Counter now accepts a name variable to help distinguish between multiple counters, time precision can be set with the 'precision' kwarg and the new attribute end_time holds the time the last iteration completed. ( #278 via Mathew Savage ) Bug Fixes Fixed an deprecation error that occured with McNemar test when using SciPy 1.0. ( #283 ) Version 0.9.0 (2017-10-21) Downloads Source code (zip) Source code (tar.gz) New Features Added evaluate.permutation_test , a permutation test for hypothesis testing (or A/B testing) to test if two samples come from the same distribution. Or in other words, a procedure to test the null hypothesis that that two groups are not significantly different (e.g., a treatment and a control group). ( #250 ) Added 'leverage' and 'conviction as evaluation metrics to the frequent_patterns.association_rules function. ( #246 & #247 ) Added a loadings_ attribute to PrincipalComponentAnalysis to compute the factor loadings of the features on the principal components. ( #251 ) Allow grid search over classifiers/regressors in ensemble and stacking estimators. ( #259 ) New make_multiplexer_dataset function that creates a dataset generated by a n-bit Boolean multiplexer for evaluating supervised learning algorithms. ( #263 ) Added a new BootstrapOutOfBag class, an implementation of the out-of-bag bootstrap to evaluate supervised learning algorithms. ( #265 ) The parameters for StackingClassifier , StackingCVClassifier , StackingRegressor , StackingCVRegressor , and EnsembleVoteClassifier can now be tuned using scikit-learn's GridSearchCV ( #254 via James Bourbeau ) Changes The 'support' column returned by frequent_patterns.association_rules was changed to compute the support of \"antecedant union consequent\", and new antecedant support' and 'consequent support' column were added to avoid ambiguity. ( #245 ) Allow the OnehotTransactions to be cloned via scikit-learn's clone function, which is required by e.g., scikit-learn's FeatureUnion or GridSearchCV (via Iaroslav Shcherbatyi ). ( #249 ) Bug Fixes Fix issues with self._init_time parameter in _IterativeModel subclasses. ( #256 ) Fix imprecision bug that occurred in plot_ecdf when run on Python 2.7. ( 264 ) The vectors from SVD in PrincipalComponentAnalysis are now being scaled so that the eigenvalues via solver='eigen' and solver='svd' now store eigenvalues that have the same magnitudes. ( #251 ) Version 0.8.0 (2017-09-09) Downloads Source code (zip) Source code (tar.gz) New Features Added a mlxtend.evaluate.bootstrap that implements the ordinary nonparametric bootstrap to bootstrap a single statistic (for example, the mean. median, R^2 of a regression fit, and so forth) #232 SequentialFeatureSelecor 's k_features now accepts a string argument \"best\" or \"parsimonious\" for more \"automated\" feature selection. For instance, if \"best\" is provided, the feature selector will return the feature subset with the best cross-validation performance. If \"parsimonious\" is provided as an argument, the smallest feature subset that is within one standard error of the cross-validation performance will be selected. #238 Changes SequentialFeatureSelector now uses np.nanmean over normal mean to support scorers that may return np.nan #211 (via mrkaiser ) The skip_if_stuck parameter was removed from SequentialFeatureSelector in favor of a more efficient implementation comparing the conditional inclusion/exclusion results (in the floating versions) to the performances of previously sampled feature sets that were cached #237 ExhaustiveFeatureSelector was modified to consume substantially less memory #195 (via Adam Erickson ) Bug Fixes Fixed a bug where the SequentialFeatureSelector selected a feature subset larger than then specified via the k_features tuple max-value #213 Version 0.7.0 (2017-06-22) Downloads Source code (zip) Source code (tar.gz) New Features New mlxtend.plotting.ecdf function for plotting empirical cumulative distribution functions ( #196 ). New StackingCVRegressor for stacking regressors with out-of-fold predictions to prevent overfitting ( #201 via Eike Dehling ). Changes The TensorFlow estimator have been removed from mlxtend, since TensorFlow has now very convenient ways to build on estimators, which render those implementations obsolete. plot_decision_regions now supports plotting decision regions for more than 2 training features #189 , via James Bourbeau ). Parallel execution in mlxtend.feature_selection.SequentialFeatureSelector and mlxtend.feature_selection.ExhaustiveFeatureSelector is now performed over different feature subsets instead of the different cross-validation folds to better utilize machines with multiple processors if the number of features is large ( #193 , via @whalebot-helmsman ). Raise meaningful error messages if pandas DataFrame s or Python lists of lists are fed into the StackingCVClassifer as a fit arguments ( 198 ). The n_folds parameter of the StackingCVClassifier was changed to cv and can now accept any kind of cross validation technique that is available from scikit-learn. For example, StackingCVClassifier(..., cv=StratifiedKFold(n_splits=3)) or StackingCVClassifier(..., cv=GroupKFold(n_splits=3)) ( #203 , via Konstantinos Paliouras ). Bug Fixes SequentialFeatureSelector now correctly accepts a None argument for the scoring parameter to infer the default scoring metric from scikit-learn classifiers and regressors ( #171 ). The plot_decision_regions function now supports pre-existing axes objects generated via matplotlib's plt.subplots . ( #184 , see example ) Made math.num_combinations and math.num_permutations numerically stable for large numbers of combinations and permutations ( #200 ). Version 0.6.0 (2017-03-18) Downloads Source code (zip) Source code (tar.gz) New Features An association_rules function is implemented that allows to generate rules based on a list of frequent itemsets (via Joshua Goerner ). Changes Adds a black edgecolor to plots via plotting.plot_decision_regions to make markers more distinguishable from the background in matplotlib>=2.0 . The association submodule was renamed to frequent_patterns . Bug Fixes The DataFrame index of apriori results are now unique and ordered. Fixed typos in autompg and wine datasets (via James Bourbeau ). Version 0.5.1 (2017-02-14) Downloads Source code (zip) Source code (tar.gz) New Features The EnsembleVoteClassifier has a new refit attribute that prevents refitting classifiers if refit=False to save computational time. Added a new lift_score function in evaluate to compute lift score (via Batuhan Bardak ). StackingClassifier and StackingRegressor support multivariate targets if the underlying models do (via kernc ). StackingClassifier has a new use_features_in_secondary attribute like StackingCVClassifier . Changes Changed default verbosity level in SequentialFeatureSelector to 0 The EnsembleVoteClassifier now raises a NotFittedError if the estimator wasn't fit before calling predict . (via Anton Loss ) Added new TensorFlow variable initialization syntax to guarantee compatibility with TensorFlow 1.0 Bug Fixes Fixed wrong default value for k_features in SequentialFeatureSelector Cast selected feature subsets in the SequentialFeautureSelector as sets to prevent the iterator from getting stuck if the k_idx are different permutations of the same combination (via Zac Wellmer ). Fixed an issue with learning curves that caused the performance metrics to be reversed (via ipashchenko ) Fixed a bug that could occur in the SequentialFeatureSelector if there are similarly-well performing subsets in the floating variants (via Zac Wellmer ). Version 0.5.0 (2016-11-09) Downloads Source code (zip) Source code (tar.gz) New Features New ExhaustiveFeatureSelector estimator in mlxtend.feature_selection for evaluating all feature combinations in a specified range The StackingClassifier has a new parameter average_probas that is set to True by default to maintain the current behavior. A deprecation warning was added though, and it will default to False in future releases (0.6.0); average_probas=False will result in stacking of the level-1 predicted probabilities rather than averaging these. New StackingCVClassifier estimator in 'mlxtend.classifier' for implementing a stacking ensemble that uses cross-validation techniques for training the meta-estimator to avoid overfitting ( Reiichiro Nakano ) New OnehotTransactions encoder class added to the preprocessing submodule for transforming transaction data into a one-hot encoded array The SequentialFeatureSelector estimator in mlxtend.feature_selection now is safely stoppable mid-process by control+c, and deprecated print_progress in favor of a more tunable verbose parameter ( Will McGinnis ) New apriori function in association to extract frequent itemsets from transaction data for association rule mining New checkerboard_plot function in plotting to plot checkerboard tables / heat maps New mcnemar_table and mcnemar functions in evaluate to compute 2x2 contingency tables and McNemar's test Changes All plotting functions have been moved to mlxtend.plotting for compatibility reasons with continuous integration services and to make the installation of matplotlib optional for users of mlxtend 's core functionality Added a compatibility layer for scikit-learn 0.18 using the new model_selection module while maintaining backwards compatibility to scikit-learn 0.17. Bug Fixes mlxtend.plotting.plot_decision_regions now draws decision regions correctly if more than 4 class labels are present Raise AttributeError in plot_decision_regions when the X_higlight argument is a 1D array ( chkoar ) Version 0.4.2 (2016-08-24) Downloads Source code (zip) Source code (tar.gz) PDF documentation New Features Added preprocessing.CopyTransformer , a mock class that returns copies of imput arrays via transform and fit_transform Changes Added AppVeyor to CI to ensure MS Windows compatibility Dataset are now saved as compressed .txt or .csv files rather than being imported as Python objects feature_selection.SequentialFeatureSelector now supports the selection of k_features using a tuple to specify a \"min-max\" k_features range Added \"SVD solver\" option to the PrincipalComponentAnalysis Raise a AttributeError with \"not fitted\" message in SequentialFeatureSelector if transform or get_metric_dict are called prior to fit Use small, positive bias units in TfMultiLayerPerceptron 's hidden layer(s) if the activations are ReLUs in order to avoid dead neurons Added an optional clone_estimator parameter to the SequentialFeatureSelector that defaults to True , avoiding the modification of the original estimator objects More rigorous type and shape checks in the evaluate.plot_decision_regions function DenseTransformer now doesn't raise and error if the input array is not sparse API clean-up using scikit-learn's BaseEstimator as parent class for feature_selection.ColumnSelector Bug Fixes Fixed a problem when a tuple-range was provided as argument to the SequentialFeatureSelector 's k_features parameter and the scoring metric was more negative than -1 (e.g., as in scikit-learn's MSE scoring function) (wahutch](https://github.com/wahutch)) Fixed an AttributeError issue when verbose > 1 in StackingClassifier Fixed a bug in classifier.SoftmaxRegression where the mean values of the offsets were used to update the bias units rather than their sum Fixed rare bug in MLP _layer_mapping functions that caused a swap between the random number generation seed when initializing weights and biases Version 0.4.1 (2016-05-01) Downloads Source code (zip) Source code (tar.gz) PDF documentation New Features New TensorFlow estimator for Linear Regression ( tf_regressor.TfLinearRegression ) New k-means clustering estimator ( cluster.Kmeans ) New TensorFlow k-means clustering estimator ( tf_cluster.Kmeans ) Changes Due to refactoring of the estimator classes, the init_weights parameter of the fit methods was globally renamed to init_params Overall performance improvements of estimators due to code clean-up and refactoring Added several additional checks for correct array types and more meaningful exception messages Added optional dropout to the tf_classifier.TfMultiLayerPerceptron classifier for regularization Added an optional decay parameter to the tf_classifier.TfMultiLayerPerceptron classifier for adaptive learning via an exponential decay of the learning rate eta Replaced old NeuralNetMLP by more streamlined MultiLayerPerceptron ( classifier.MultiLayerPerceptron ); now also with softmax in the output layer and categorical cross-entropy loss. Unified init_params parameter for fit functions to continue training where the algorithm left off (if supported) Version 0.4.0 (2016-04-09) New Features New TfSoftmaxRegression classifier using Tensorflow ( tf_classifier.TfSoftmaxRegression ) New SoftmaxRegression classifier ( classifier.SoftmaxRegression ) New TfMultiLayerPerceptron classifier using Tensorflow ( tf_classifier.TfMultiLayerPerceptron ) New StackingRegressor ( regressor.StackingRegressor ) New StackingClassifier ( classifier.StackingClassifier ) New function for one-hot encoding of class labels ( preprocessing.one_hot ) Added GridSearch support to the SequentialFeatureSelector ( feature_selection/.SequentialFeatureSelector ) evaluate.plot_decision_regions improvements: Function now handles class y-class labels correctly if array is of type float Correct handling of input arguments markers and colors Accept an existing Axes via the ax argument New print_progress parameter for all generalized models and multi-layer neural networks for printing time elapsed, ETA, and the current cost of the current epoch Minibatch learning for classifier.LogisticRegression , classifier.Adaline , and regressor.LinearRegression plus streamlined API New Principal Component Analysis class via mlxtend.feature_extraction.PrincipalComponentAnalysis New RBF Kernel Principal Component Analysis class via mlxtend.feature_extraction.RBFKernelPCA New Linear Discriminant Analysis class via mlxtend.feature_extraction.LinearDiscriminantAnalysis Changes The column parameter in mlxtend.preprocessing.standardize now defaults to None to standardize all columns more conveniently Version 0.3.0 (2016-01-31) Downloads Source code (zip) Source code (tar.gz) New Features Added a progress bar tracker to classifier.NeuralNetMLP Added a function to score predicted vs. target class labels evaluate.scoring Added confusion matrix functions to create ( evaluate.confusion_matrix ) and plot ( evaluate.plot_confusion_matrix ) confusion matrices New style parameter and improved axis scaling in mlxtend.evaluate.plot_learning_curves Added loadlocal_mnist to mlxtend.data for streaming MNIST from a local byte files into numpy arrays New NeuralNetMLP parameters: random_weights , shuffle_init , shuffle_epoch New SFS features such as the generation of pandas DataFrame results tables and plotting functions (with confidence intervals, standard deviation, and standard error bars) Added support for regression estimators in SFS Added Boston housing dataset New shuffle parameter for classifier.NeuralNetMLP Changes The mlxtend.preprocessing.standardize function now optionally returns the parameters, which are estimated from the array, for re-use. A further improvement makes the standardize function smarter in order to avoid zero-division errors Cosmetic improvements to the evaluate.plot_decision_regions function such as hiding plot axes Renaming of classifier.EnsembleClassfier to classifier.EnsembleVoteClassifier Improved random weight initialization in Perceptron , Adaline , LinearRegression , and LogisticRegression Changed learning parameter of mlxtend.classifier.Adaline to solver and added \"normal equation\" as closed-form solution solver Hide y-axis labels in mlxtend.evaluate.plot_decision_regions in 1 dimensional evaluations Sequential Feature Selection algorithms were unified into a single SequentialFeatureSelector class with parameters to enable floating selection and toggle between forward and backward selection. Stratified sampling of MNIST (now 500x random samples from each of the 10 digit categories) Renaming mlxtend.plotting to mlxtend.general_plotting in order to distinguish general plotting function from specialized utility function such as evaluate.plot_decision_regions Version 0.2.9 (2015-07-14) Downloads Source code (zip) Source code (tar.gz) New Features Sequential Feature Selection algorithms: SFS, SFFS, SBS, and SFBS Changes Changed regularization & lambda parameters in LogisticRegression to single parameter l2_lambda Version 0.2.8 (2015-06-27) API changes: mlxtend.sklearn.EnsembleClassifier -> mlxtend.classifier.EnsembleClassifier mlxtend.sklearn.ColumnSelector -> mlxtend.feature_selection.ColumnSelector mlxtend.sklearn.DenseTransformer -> mlxtend.preprocessing.DenseTransformer mlxtend.pandas.standardizing -> mlxtend.preprocessing.standardizing mlxtend.pandas.minmax_scaling -> mlxtend.preprocessing.minmax_scaling mlxtend.matplotlib -> mlxtend.plotting Added momentum learning parameter (alpha coefficient) to mlxtend.classifier.NeuralNetMLP . Added adaptive learning rate (decrease constant) to mlxtend.classifier.NeuralNetMLP . mlxtend.pandas.minmax_scaling became mlxtend.preprocessing.minmax_scaling and also supports NumPy arrays now mlxtend.pandas.standardizing became mlxtend.preprocessing.standardizing and now supports both NumPy arrays and pandas DataFrames; also, now ddof parameters to set the degrees of freedom when calculating the standard deviation Version 0.2.7 (2015-06-20) Added multilayer perceptron (feedforward artificial neural network) classifier as mlxtend.classifier.NeuralNetMLP . Added 5000 labeled trainingsamples from the MNIST handwritten digits dataset to mlxtend.data Version 0.2.6 (2015-05-08) Added ordinary least square regression using different solvers (gradient and stochastic gradient descent, and the closed form solution (normal equation) Added option for random weight initialization to logistic regression classifier and updated l2 regularization Added wine dataset to mlxtend.data Added invert_axes parameter mlxtend.matplotlib.enrichtment_plot to optionally plot the \"Count\" on the x-axis New verbose parameter for mlxtend.sklearn.EnsembleClassifier by Alejandro C. Bahnsen Added mlxtend.pandas.standardizing to standardize columns in a Pandas DataFrame Added parameters linestyles and markers to mlxtend.matplotlib.enrichment_plot mlxtend.regression.lin_regplot automatically adds np.newaxis and works w. python lists Added tokenizers: mlxtend.text.extract_emoticons and mlxtend.text.extract_words_and_emoticons Version 0.2.5 (2015-04-17) Added Sequential Backward Selection (mlxtend.sklearn.SBS) Added X_highlight parameter to mlxtend.evaluate.plot_decision_regions for highlighting test data points. Added mlxtend.regression.lin_regplot to plot the fitted line from linear regression. Added mlxtend.matplotlib.stacked_barplot to conveniently produce stacked barplots using pandas DataFrame s. Added mlxtend.matplotlib.enrichment_plot Version 0.2.4 (2015-03-15) Added scoring to mlxtend.evaluate.learning_curves (by user pfsq) Fixed setup.py bug caused by the missing README.html file matplotlib.category_scatter for pandas DataFrames and Numpy arrays Version 0.2.3 (2015-03-11) Added Logistic regression Gradient descent and stochastic gradient descent perceptron was changed to Adaline (Adaptive Linear Neuron) Perceptron and Adaline for {0, 1} classes Added mlxtend.preprocessing.shuffle_arrays_unison function to shuffle one or more NumPy arrays. Added shuffle and random seed parameter to stochastic gradient descent classifier. Added rstrip parameter to mlxtend.file_io.find_filegroups to allow trimming of base names. Added ignore_substring parameter to mlxtend.file_io.find_filegroups and find_files . Replaced .rstrip in mlxtend.file_io.find_filegroups with more robust regex. Gridsearch support for mlxtend.sklearn.EnsembleClassifier Version 0.2.2 (2015-03-01) Improved robustness of EnsembleClassifier. Extended plot_decision_regions() functionality for plotting 1D decision boundaries. Function matplotlib.plot_decision_regions was reorganized to evaluate.plot_decision_regions . evaluate.plot_learning_curves() function added. Added Rosenblatt, gradient descent, and stochastic gradient descent perceptrons. Version 0.2.1 (2015-01-20) Added mlxtend.pandas.minmax_scaling - a function to rescale pandas DataFrame columns. Slight update to the EnsembleClassifier interface (additional voting parameter) Fixed EnsembleClassifier to return correct class labels if class labels are not integers from 0 to n. Added new matplotlib function to plot decision regions of classifiers. Version 0.2.0 (2015-01-13) Improved mlxtend.text.generalize_duplcheck to remove duplicates and prevent endless looping issue. Added recursive search parameter to mlxtend.file_io.find_files. Added check_ext parameter mlxtend.file_io.find_files to search based on file extensions. Default parameter to ignore invisible files for mlxtend.file_io.find. Added transform and fit_transform to the EnsembleClassifier . Added mlxtend.file_io.find_filegroups function. Version 0.1.9 (2015-01-10) Implemented scikit-learn EnsembleClassifier (majority voting rule) class. Version 0.1.8 (2015-01-07) Improvements to mlxtend.text.generalize_names to handle certain Dutch last name prefixes (van, van der, de, etc.). Added mlxtend.text.generalize_name_duplcheck function to apply mlxtend.text.generalize_names function to a pandas DataFrame without creating duplicates. Version 0.1.7 (2015-01-07) Added text utilities with name generalization function. Added and file_io utilities. Version 0.1.6 (2015-01-04) Added combinations and permutations estimators. Version 0.1.5 (2014-12-11) Added DenseTransformer for pipelines and grid search. Version 0.1.4 (2014-08-20) mean_centering function is now a Class that creates MeanCenterer objects that can be used to fit data via the fit method, and center data at the column means via the transform and fit_transform method. Version 0.1.3 (2014-08-19) Added preprocessing module and mean_centering function. Version 0.1.2 (2014-08-19) Added matplotlib utilities and remove_borders function. Version 0.1.1 (2014-08-13) Simplified code for ColumnSelector.","title":"Release Notes"},{"location":"CHANGELOG/#release-notes","text":"The CHANGELOG for the current development version is available at https://github.com/rasbt/mlxtend/blob/master/docs/sources/CHANGELOG.md .","title":"Release Notes"},{"location":"CHANGELOG/#version-0233-15-nov-2024","text":"","title":"Version 0.23.3  (15 Nov 2024)"},{"location":"CHANGELOG/#downloads","text":"Source code (zip) Source code (tar.gz)","title":"Downloads"},{"location":"CHANGELOG/#new-features-and-enhancements","text":"Files updated: - mlxtend.evaluate.time_series.plot_splits - Improved plot_splits for better visualization of time series splits","title":"New Features and Enhancements"},{"location":"CHANGELOG/#changes","text":"mlxtend/feature_selection/exhaustive_feature_selector.py np.inf update to support for NumPy 2.0","title":"Changes"},{"location":"CHANGELOG/#version-0232-5-nov-2024","text":"","title":"Version 0.23.2  (5 Nov 2024)"},{"location":"CHANGELOG/#downloads_1","text":"Source code (zip) Source code (tar.gz)","title":"Downloads"},{"location":"CHANGELOG/#new-features-and-enhancements_1","text":"Implement the FP-Growth and FP-Max algorithms with the possibility of missing values in the input dataset. Added a new metric Representativity for the association rules generated ( #1004 via zazass8 ). Files updated: ['mlxtend.frequent_patterns.fpcommon'] 'mlxtend.frequent_patterns.fpgrowth' 'mlxtend.frequent_patterns.fpmax' 'mlxtend/feature_selection/utilities.py' Modified _calc_score function to ensure compatibility with scikit-learn versions 1.4 and above by dynamically selecting between fit_params and params in cross_val_score . mlxtend.feature_selection.SequentialFeatureSelector Updated negative infinity constant to be compatible with old and new (>=2.0) numpy versions mlxtend.frequent_patterns.association_rules Implemented three new metrics: Jaccard, Certainty, and Kulczynski. ( #1096 ) Integrated scikit-learn's set_output method into TransactionEncoder ( #1087 via it176131 )","title":"New Features and Enhancements"},{"location":"CHANGELOG/#changes_1","text":"[ mlxtend.frequent_patterns.fpcommon ] Added the null_values parameter in valid_input_check signature to check in case the input also includes null values. Changes the returns statements and function signatures for setup_fptree and generated_itemsets respectively to return the disabled array created and to include it as a parameter. Added code in [ mlxtend.frequent_patterns.fpcommon ] and mlxtend.frequent_patterns.association_rules to implement the algorithms in case null values exist when null_values is True. mlxtend.frequent_patterns.association_rules Added optional parameter 'return_metrics' to only return a given list of metrics, rather than every possible metric. Add n_classes_ attribute to stacking classifiers for compatibility with scikit-learn 1.3 ( #1091 ) Use Scipy's instead of NumPy's decompositions in PCA for improved accuracy in edge cases ( #1080 via [ fkdosilovic ])","title":"Changes"},{"location":"CHANGELOG/#version-0231-5-jan-2024","text":"","title":"Version 0.23.1 (5 Jan 2024)"},{"location":"CHANGELOG/#downloads_2","text":"Source code (zip) Source code (tar.gz)","title":"Downloads"},{"location":"CHANGELOG/#changes_2","text":"Updated dependency on distutils for python 3.12 and above ( #1072 via peanutsee )","title":"Changes"},{"location":"CHANGELOG/#version-0230-22-sep-2023","text":"","title":"Version 0.23.0 (22 Sep 2023)"},{"location":"CHANGELOG/#downloads_3","text":"Source code (zip) Source code (tar.gz)","title":"Downloads"},{"location":"CHANGELOG/#changes_3","text":"Address NumPy deprecations to make mlxtend compatible to NumPy 1.24 Changed the signature of the LinearRegression model of sklearn in the test removing the normalize parameter as it is deprecated. ( #1036 ) Add pyproject.toml to support PEP 518 builds ( #1065 via jmahlik ) Fixed installation from sdist failing ( #1065 via jmahlik ) Converted configuration to pyproject.toml ( #1065 via jmahlik ) Remove mlxtend.image submodule with face recognition functions due to poor dlib support in modern environments.","title":"Changes"},{"location":"CHANGELOG/#new-features-and-enhancements_2","text":"Document how to use SequentialFeatureSelector and multiclass ROC AUC.","title":"New Features and Enhancements"},{"location":"CHANGELOG/#version-0220-4-april-2023","text":"","title":"Version 0.22.0 (4 April 2023)"},{"location":"CHANGELOG/#downloads_4","text":"Source code (zip) Source code (tar.gz)","title":"Downloads"},{"location":"CHANGELOG/#changes_4","text":"When ExhaustiveFeatureSelector is run with n_jobs == 1 , joblib is now disabled, which enables more immediate (live) feedback when the verbose mode is enabled. ( #985 via Nima Sarajpoor ) Disabled unnecessary warning in EnsembleVoteClassifier ( #941 ) Fixed various documentation issues ( #849 and #951 via Lekshmanan Natarajan ) Fixed \"Edit on GitHub\" button ( #1024 )","title":"Changes"},{"location":"CHANGELOG/#new-features-and-enhancements_3","text":"The mlxtend.frequent_patterns.association_rules function has a new metric - Zhang's Metric, which measures both association and dissociation. ( #980 ) Internal mlxtend.frequent_patterns.fpmax code improvement that avoids casting a sparse DataFrame into a dense NumPy array. ( #1000 via Tim Kellogg ) The plot_decision_regions function now has a n_jobs parameter to parallelize the computation. (In a particular use case, on a small dataset, there was a 21x speed-up (449 seconds vs 21 seconds on local HPC instance of 36 cores). ( #998 via Khalid ElHaj ) Added mlxtend.frequent_patterns.hmine algorithm and documentation for mining frequent itemsets using the H-Mine algorithm. ( #1020 via Fatih Sen )","title":"New Features and Enhancements"},{"location":"CHANGELOG/#version-0210-09172022","text":"","title":"Version 0.21.0 (09/17/2022)"},{"location":"CHANGELOG/#downloads_5","text":"Source code (zip) Source code (tar.gz)","title":"Downloads"},{"location":"CHANGELOG/#new-features-and-enhancements_4","text":"The mlxtend.evaluate.feature_importance_permutation function has a new feature_groups argument to treat user-specified feature groups as single features, which is useful for one-hot encoded features. ( #955 ) The mlxtend.feature_selection.ExhaustiveFeatureSelector and SequentialFeatureSelector also gained support for feature_groups with a behavior similar to the one described above. ( #957 and #965 via Nima Sarajpoor )","title":"New Features and Enhancements"},{"location":"CHANGELOG/#changes_5","text":"The custom_feature_names parameter was removed from the ExhaustiveFeatureSelector due to redundancy and to simplify the code base. The ExhaustiveFeatureSelector documentation illustrates how the same behavior and outcome can be achieved using pandas DataFrames. ( #957 )","title":"Changes"},{"location":"CHANGELOG/#bug-fixes","text":"None","title":"Bug Fixes"},{"location":"CHANGELOG/#version-0200-05262022","text":"","title":"Version 0.20.0 (05/26/2022)"},{"location":"CHANGELOG/#downloads_6","text":"Source code (zip) Source code (tar.gz)","title":"Downloads"},{"location":"CHANGELOG/#new-features-and-enhancements_5","text":"The mlxtend.evaluate.bootstrap_point632_score now supports fit_params . ( #861 ) The mlxtend/plotting/decision_regions.py function now has a contourf_kwargs for matplotlib to change the look of the decision boundaries if desired. ( #881 via [ pbloem ]) Add a norm_colormap parameter to mlxtend.plotting.plot_confusion_matrix , to allow normalizing the colormap, e.g., using matplotlib.colors.LogNorm() ( #895 ) Add new GroupTimeSeriesSplit class for evaluation in time series tasks with support of custom groups and additional parameters in comparison with scikit-learn's TimeSeriesSplit . ( #915 via Dmitry Labazkin )","title":"New Features and Enhancements"},{"location":"CHANGELOG/#changes_6","text":"Due to compatibility issues with newer package versions, certain functions from six.py have been removed so that mlxtend may not work anymore with Python 2.7. As an internal change to speed up unit testing, unit testing is now faciliated by GitHub workflows, and Travis CI and Appveyor hooks have been removed. Improved axis label rotation in mlxtend.plotting.heatmap and mlxtend.plotting.plot_confusion_matrix ( #872 ) Fix various typos in McNemar guides. Raises a warning if non-bool arrays are used in the frequent pattern functions apriori , fpmax , and fpgrowth . ( #934 via NimaSarajpoor )","title":"Changes"},{"location":"CHANGELOG/#bug-fixes_1","text":"Fix unreadable labels in heatmap for certain colormaps. ( #852 ) Fix an issue in mlxtend.plotting.plot_confusion_matrix when string class names are passed ( #894 )","title":"Bug Fixes"},{"location":"CHANGELOG/#version-0190-2021-09-02","text":"","title":"Version 0.19.0 (2021-09-02)"},{"location":"CHANGELOG/#downloads_7","text":"Source code (zip) Source code (tar.gz)","title":"Downloads"},{"location":"CHANGELOG/#new-features","text":"Adds a second \"balanced accuracy\" interpretation (\"balanced\") to evaluate.accuracy_score in addition to the existing \"average\" option to compute the scikit-learn-style balanced accuracy. ( #764 ) Adds new scatter_hist function to mlxtend.plotting for generating a scattered histogram. ( #757 via Maitreyee Mhasaka ) The evaluate.permutation_test function now accepts a paired argument to specify to support paired permutation/randomization tests. ( #768 ) The StackingCVRegressor now also supports multi-dimensional targets similar to StackingRegressor via StackingCVRegressor(..., multi_output=True) . ( #802 via Marco Tiraboschi )","title":"New Features"},{"location":"CHANGELOG/#changes_7","text":"Updates unit tests for scikit-learn 0.24.1 compatibility. ( #774 ) StackingRegressor now requires setting StackingRegressor(..., multi_output=True) if the target is multi-dimensional; this allows for better input validation. ( #802 ) Removes deprecated res argument from plot_decision_regions . ( #803 ) Adds a title_fontsize parameter to plot_learning_curves for controlling the title font size; also the plot style is now the matplotlib default. ( #818 ) Internal change using 'c': 'none' instead of 'c': '' in mlxtend.plotting.plot_decision_regions 's scatterplot highlights to stay compatible with Matplotlib 3.4 and newer. ( #822 ) Adds a fontcolor_threshold parameter to the mlxtend.plotting.plot_confusion_matrix function as an additional option for determining the font color cut-off manually. ( #827 ) The frequent_patterns.association_rules now raises a ValueError if an empty frequent itemset DataFrame is passed. ( #843 ) The .632 and .632+ bootstrap method implemented in the mlxtend.evaluate.bootstrap_point632_score function now use the whole training set for the resubstitution weighting term instead of the internal training set that is a new bootstrap sample in each round. ( #844 )","title":"Changes"},{"location":"CHANGELOG/#bug-fixes_2","text":"Fixes a typo in the SequentialFeatureSelector documentation ( #835 via Jo\u00e3o Pedro Zanlorensi Cardoso )","title":"Bug Fixes"},{"location":"CHANGELOG/#version-0180-2020-11-25","text":"","title":"Version 0.18.0 (2020-11-25)"},{"location":"CHANGELOG/#downloads_8","text":"Source code (zip) Source code (tar.gz)","title":"Downloads"},{"location":"CHANGELOG/#new-features_1","text":"The bias_variance_decomp function now supports optional fit_params for the estimators that are fit on bootstrap samples. ( #748 ) The bias_variance_decomp function now supports Keras estimators. ( #725 via @hanzigs ) Adds new mlxtend.classifier.OneRClassifier (One Rule Classfier) class, a simple rule-based classifier that is often used as a performance baseline or simple interpretable model. ( #726 Adds new create_counterfactual method for creating counterfactuals to explain model predictions. ( #740 )","title":"New Features"},{"location":"CHANGELOG/#changes_8","text":"permutation_test ( mlxtend.evaluate.permutation ) \u00ecs corrected to give the proportion of permutations whose statistic is at least as extreme as the one observed. ( #721 via Florian Charlier ) Fixes the McNemar confusion matrix layout to match the convention (and documentation), swapping the upper left and lower right cells. ( #744 via mmarius )","title":"Changes"},{"location":"CHANGELOG/#bug-fixes_3","text":"The loss in LogisticRegression for logging purposes didn't include the L2 penalty for the first weight in the weight vector (this is not the bias unit). However, since this loss function was only used for logging purposes, and the gradient remains correct, this does not have an effect on the main code. ( #741 ) Fixes a bug in bias_variance_decomp where when the mse loss was used, downcasting to integers caused imprecise results for small numbers. ( #749 )","title":"Bug Fixes"},{"location":"CHANGELOG/#version-0173-2020-07-27","text":"","title":"Version 0.17.3 (2020-07-27)"},{"location":"CHANGELOG/#downloads_9","text":"Source code (zip) Source code (tar.gz)","title":"Downloads"},{"location":"CHANGELOG/#new-features_2","text":"Add predict_proba kwarg to bootstrap methods, to allow bootstrapping of scoring functions that take in probability values. ( #700 via Adam Li ) Add a cell_values parameter to mlxtend.plotting.heatmap() to optionally suppress cell annotations by setting cell_values=False . ( #703","title":"New Features"},{"location":"CHANGELOG/#changes_9","text":"Implemented both use_clones and fit_base_estimators (previously refit in EnsembleVoteClassifier ) for EnsembleVoteClassifier and StackingClassifier . ( #670 via Katrina Ni ) Switched to using raw strings for regex in mlxtend.text to prevent deprecation warning in Python 3.8 ( #688 ) Slice data in sequential forward selection before sending to parallel backend, reducing memory consumption.","title":"Changes"},{"location":"CHANGELOG/#bug-fixes_4","text":"Fixes axis DeprecationWarning in matplotlib v3.1.0 and newer. ( #673 ) Fixes an issue with using meshgrid in no_information_rate function used by the bootstrap_point632_score function for the .632+ estimate. ( #688 ) Fixes an issue in fpmax that could lead to incorrect support values. ( #692 via Steve Harenberg )","title":"Bug Fixes"},{"location":"CHANGELOG/#version-0172-2020-02-24","text":"","title":"Version 0.17.2 (2020-02-24)"},{"location":"CHANGELOG/#downloads_10","text":"Source code (zip) Source code (tar.gz)","title":"Downloads"},{"location":"CHANGELOG/#new-features_3","text":"-","title":"New Features"},{"location":"CHANGELOG/#changes_10","text":"The previously deprecated OnehotTransactions has been removed in favor of the TransactionEncoder. Removed SparseDataFrame support in frequent pattern mining functions in favor of pandas >=1.0's new way for working sparse data. If you used SparseDataFrame formats, please see pandas' migration guide at https://pandas.pydata.org/pandas-docs/stable/user_guide/sparse.html#migrating. ( #667 ) The plot_confusion_matrix.py now also accepts a matplotlib figure and axis as input to which the confusion matrix plot can be added. ( #671 via Vahid Mirjalili )","title":"Changes"},{"location":"CHANGELOG/#bug-fixes_5","text":"-","title":"Bug Fixes"},{"location":"CHANGELOG/#version-0171-2020-01-28","text":"","title":"Version 0.17.1 (2020-01-28)"},{"location":"CHANGELOG/#downloads_11","text":"Source code (zip) Source code (tar.gz)","title":"Downloads"},{"location":"CHANGELOG/#new-features_4","text":"The SequentialFeatureSelector now supports using pre-specified feature sets via the fixed_features parameter. ( #578 ) Adds a new accuracy_score function to mlxtend.evaluate for computing basic classifcation accuracy, per-class accuracy, and average per-class accuracy. ( #624 via Deepan Das ) StackingClassifier and StackingCVClassifier now have a decision_function method, which serves as a preferred choice over predict_proba in calculating roc_auc and average_precision scores when the meta estimator is a linear model or support vector classifier. ( #634 via Qiang Gu )","title":"New Features"},{"location":"CHANGELOG/#changes_11","text":"Improve the runtime performance for the apriori frequent itemset generating function when low_memory=True . Setting low_memory=False (default) is still faster for small itemsets, but low_memory=True can be much faster for large itemsets and requires less memory. Also, input validation for apriori , \u0300 fpgrowth and fpmax takes a significant amount of time when input pandas DataFrame is large; this is now dramatically reduced when input contains boolean values (and not zeros/ones), which is the case when using TransactionEncoder`. ( #619 via Denis Barbier ) Add support for newer sparse pandas DataFrame for frequent itemset algorithms. Also, input validation for apriori , \u0300 fpgrowth and fpmax` runs much faster on sparse DataFrame when input pandas DataFrame contains integer values. ( #621 via Denis Barbier ) Let fpgrowth and fpmax directly work on sparse DataFrame, they were previously converted into dense Numpy arrays. ( #622 via Denis Barbier )","title":"Changes"},{"location":"CHANGELOG/#bug-fixes_6","text":"Fixes a bug in mlxtend.plotting.plot_pca_correlation_graph that caused the explaind variances not summing up to 1. Also, improves the runtime performance of the correlation computation and adds a missing function argument for the explained variances (eigenvalues) if users provide their own principal components. ( #593 via Gabriel Azevedo Ferreira ) Behavior of fpgrowth and apriori consistent for edgecases such as min_support=0 . ( #573 via Steve Harenberg ) fpmax returns an empty data frame now instead of raising an error if the frequent itemset set is empty. ( #573 via Steve Harenberg ) Fixes and issue in mlxtend.plotting.plot_confusion_matrix , where the font-color choice for medium-dark cells was not ideal and hard to read. #588 via sohrabtowfighi ) The svd mode of mlxtend.feature_extraction.PrincipalComponentAnalysis now also n-1 degrees of freedom instead of n d.o.f. when computing the eigenvalues to match the behavior of eigen . #595 Disable input validation for StackingCVClassifier because it causes issues if pipelines are used as input. #606","title":"Bug Fixes"},{"location":"CHANGELOG/#version-0170-2019-07-19","text":"","title":"Version 0.17.0 (2019-07-19)"},{"location":"CHANGELOG/#downloads_12","text":"Source code (zip) Source code (tar.gz)","title":"Downloads"},{"location":"CHANGELOG/#new-features_5","text":"Added an enhancement to the existing iris_data() such that both the UCI Repository version of the Iris dataset as well as the corrected, original version of the dataset can be loaded, which has a slight difference in two data points (consistent with Fisher's paper; this is also the same as in R). (via #539 via janismdhanbad ) Added optional groups parameter to SequentialFeatureSelector and ExhaustiveFeatureSelector fit() methods for forwarding to sklearn CV ( #537 via arc12 ) Added a new plot_pca_correlation_graph function to the mlxtend.plotting submodule for plotting a PCA correlation graph. ( #544 via Gabriel-Azevedo-Ferreira ) Added a zoom_factor parameter to the mlxten.plotting.plot_decision_region function that allows users to zoom in and out of the decision region plots. ( #545 ) Added a function fpgrowth that implements the FP-Growth algorithm for mining frequent itemsets as a drop-in replacement for the existing apriori algorithm. ( #550 via Steve Harenberg ) New heatmap function in mlxtend.plotting . ( #552 ) Added a function fpmax that implements the FP-Max algorithm for mining maximal itemsets as a drop-in replacement for the fpgrowth algorithm. ( #553 via Steve Harenberg ) New figsize parameter for the plot_decision_regions function in mlxtend.plotting . ( #555 via Mirza Hasanbasic ) New low_memory option for the apriori frequent itemset generating function. Setting low_memory=False (default) uses a substantially optimized version of the algorithm that is 3-6x faster than the original implementation ( low_memory=True ). ( #567 via jmayse ) Added numerically stable OLS methods which uses QR decomposition and Singular Value Decomposition (SVD) methods to LinearRegression in mlxtend.regressor.linear_regression . ( #575 via PuneetGrov3r )","title":"New Features"},{"location":"CHANGELOG/#changes_12","text":"Now uses the latest joblib library under the hood for multiprocessing instead of sklearn.externals.joblib . ( #547 ) Changes to StackingCVClassifier and StackingCVRegressor such that first-level models are allowed to generate output of non-numeric type. ( #562 )","title":"Changes"},{"location":"CHANGELOG/#bug-fixes_7","text":"Fixed documentation of iris_data() under iris.py by adding a note about differences in the iris data in R and UCI machine learning repo. Make sure that if the 'svd' mode is used in PCA, the number of eigenvalues is the same as when using 'eigen' (append 0's zeros in that case) ( #565 )","title":"Bug Fixes"},{"location":"CHANGELOG/#version-0160-2019-05-12","text":"","title":"Version 0.16.0 (2019-05-12)"},{"location":"CHANGELOG/#downloads_13","text":"Source code (zip) Source code (tar.gz)","title":"Downloads"},{"location":"CHANGELOG/#new-features_6","text":"StackingCVClassifier and StackingCVRegressor now support random_state parameter, which, together with shuffle , controls the randomness in the cv splitting. ( #523 via Qiang Gu ) StackingCVClassifier and StackingCVRegressor now have a new drop_last_proba parameter. It drops the last \"probability\" column in the feature set since if True , because it is redundant: p(y_c) = 1 - p(y_1) + p(y_2) + ... + p(y_{c-1}). This can be useful for meta-classifiers that are sensitive to perfectly collinear features. ( #532 ) Other stacking estimators, including StackingClassifier , StackingCVClassifier and StackingRegressor , support grid search over the regressors and even a single base regressor. ( #522 via Qiang Gu ) Adds multiprocessing support to StackingCVClassifier . ( #522 via Qiang Gu ) Adds multiprocessing support to StackingCVRegressor . ( #512 via Qiang Gu ) Now, the StackingCVRegressor also enables grid search over the regressors and even a single base regressor. When there are level-mixed parameters, GridSearchCV will try to replace hyperparameters in a top-down order (see the documentation for examples details). ( #515 via Qiang Gu ) Adds a verbose parameter to apriori to show the current iteration number as well as the itemset size currently being sampled. ( #519 Adds an optional class_name parameter to the confusion matrix function to display class names on the axis as tick marks. ( #487 via sandpiturtle ) Adds a pca.e_vals_normalized_ attribute to PCA for storing the eigenvalues also in normalized form; this is commonly referred to as variance explained ratios. #545","title":"New Features"},{"location":"CHANGELOG/#changes_13","text":"Due to new features, restructuring, and better scikit-learn support (for GridSearchCV , etc.) the StackingCVRegressor 's meta regressor is now being accessed via 'meta_regressor__* in the parameter grid. E.g., if a RandomForestRegressor as meta- egressor was previously tuned via 'randomforestregressor__n_estimators' , this has now changed to 'meta_regressor__n_estimators' . ( #515 via Qiang Gu ) The same change mentioned above is now applied to other stacking estimators, including StackingClassifier , StackingCVClassifier and StackingRegressor . ( #522 via Qiang Gu ) Automatically performs mean centering for PCA solver 'SVD' such that using SVD is always equal to using the covariance matrix approach #545","title":"Changes"},{"location":"CHANGELOG/#bug-fixes_8","text":"The feature_selection.ColumnSelector now also supports column names of type int (in addition to str names) if the input is a pandas DataFrame. ( #500 via tetrar124 Fix unreadable labels in plot_confusion_matrix for imbalanced datasets if show_absolute=True and show_normed=True . ( #504 ) Raises a more informative error if a SparseDataFrame is passed to apriori and the dataframe has integer column names that don't start with 0 due to current limitations of the SparseDataFrame implementation in pandas. ( #503 ) SequentialFeatureSelector now supports DataFrame as input for all operating modes (forward/backward/floating). #506 mlxtend.evaluate.feature_importance_permutation now correctly accepts scoring functions with proper function signature as metric argument. #528","title":"Bug Fixes"},{"location":"CHANGELOG/#version-0150-2019-01-19","text":"","title":"Version 0.15.0 (2019-01-19)"},{"location":"CHANGELOG/#downloads_14","text":"Source code (zip) Source code (tar.gz)","title":"Downloads"},{"location":"CHANGELOG/#new-features_7","text":"Adds a new transformer class to mlxtend.image , EyepadAlign , that aligns face images based on the location of the eyes. ( #466 by Vahid Mirjalili ) Adds a new function, mlxtend.evaluate.bias_variance_decomp that decomposes the loss of a regressor or classifier into bias and variance terms. ( #470 ) Adds a whitening parameter to PrincipalComponentAnalysis , to optionally whiten the transformed data such that the features have unit variance. ( #475 )","title":"New Features"},{"location":"CHANGELOG/#changes_14","text":"Changed the default solver in PrincipalComponentAnalysis to 'svd' instead of 'eigen' to improve numerical stability. ( #474 ) The mlxtend.image.extract_face_landmarks now returns None if no facial landmarks were detected instead of an array of all zeros. ( #466 )","title":"Changes"},{"location":"CHANGELOG/#bug-fixes_9","text":"The eigenvectors maybe have not been sorted in certain edge cases if solver was 'eigen' in PrincipalComponentAnalysis and LinearDiscriminantAnalysis . ( #477 , #478 )","title":"Bug Fixes"},{"location":"CHANGELOG/#version-0140-2018-11-09","text":"","title":"Version 0.14.0 (2018-11-09)"},{"location":"CHANGELOG/#downloads_15","text":"Source code (zip) Source code (tar.gz)","title":"Downloads"},{"location":"CHANGELOG/#new-features_8","text":"Added a scatterplotmatrix function to the plotting module. ( #437 ) Added sample_weight option to StackingRegressor , StackingClassifier , StackingCVRegressor , StackingCVClassifier , EnsembleVoteClassifier . ( #438 ) Added a RandomHoldoutSplit class to perform a random train/valid split without rotation in SequentialFeatureSelector , scikit-learn GridSearchCV etc. ( #442 ) Added a PredefinedHoldoutSplit class to perform a train/valid split, based on user-specified indices, without rotation in SequentialFeatureSelector , scikit-learn GridSearchCV etc. ( #443 ) Created a new mlxtend.image submodule for working on image processing-related tasks. ( #457 ) Added a new convenience function extract_face_landmarks based on dlib to mlxtend.image . ( #458 ) Added a method='oob' option to the mlxtend.evaluate.bootstrap_point632_score method to compute the classic out-of-bag bootstrap estimate ( #459 ) Added a method='.632+' option to the mlxtend.evaluate.bootstrap_point632_score method to compute the .632+ bootstrap estimate that addresses the optimism bias of the .632 bootstrap ( #459 ) Added a new mlxtend.evaluate.ftest function to perform an F-test for comparing the accuracies of two or more classification models. ( #460 ) Added a new mlxtend.evaluate.combined_ftest_5x2cv function to perform an combined 5x2cv F-Test for comparing the performance of two models. ( #461 ) Added a new mlxtend.evaluate.difference_proportions test for comparing two proportions (e.g., classifier accuracies) ( #462 )","title":"New Features"},{"location":"CHANGELOG/#changes_15","text":"Addressed deprecations warnings in NumPy 0.15. ( #425 ) Because of complications in PR ( #459 ), Python 2.7 was now dropped; since official support for Python 2.7 by the Python Software Foundation is ending in approx. 12 months anyways, this re-focussing will hopefully free up some developer time with regard to not having to worry about backward compatibility","title":"Changes"},{"location":"CHANGELOG/#bug-fixes_10","text":"Fixed an issue with a missing import in mlxtend.plotting.plot_confusion_matrix . ( #428 )","title":"Bug Fixes"},{"location":"CHANGELOG/#version-0130-2018-07-20","text":"","title":"Version 0.13.0 (2018-07-20)"},{"location":"CHANGELOG/#downloads_16","text":"Source code (zip) Source code (tar.gz)","title":"Downloads"},{"location":"CHANGELOG/#new-features_9","text":"A meaningful error message is now raised when a cross-validation generator is used with SequentialFeatureSelector . ( #377 ) The SequentialFeatureSelector now accepts custom feature names via the fit method for more interpretable feature subset reports. ( #379 ) The SequentialFeatureSelector is now also compatible with Pandas DataFrames and uses DataFrame column-names for more interpretable feature subset reports. ( #379 ) ColumnSelector now works with Pandas DataFrames columns. ( #378 by Manuel Garrido ) The ExhaustiveFeatureSelector estimator in mlxtend.feature_selection now is safely stoppable mid-process by control+c. ( #380 ) Two new functions, vectorspace_orthonormalization and vectorspace_dimensionality were added to mlxtend.math to use the Gram-Schmidt process to convert a set of linearly independent vectors into a set of orthonormal basis vectors, and to compute the dimensionality of a vectorspace, respectively. ( #382 ) mlxtend.frequent_patterns.apriori now supports pandas SparseDataFrame s to generate frequent itemsets. ( #404 via Daniel Morales ) The plot_confusion_matrix function now has the ability to show normalized confusion matrix coefficients in addition to or instead of absolute confusion matrix coefficients with or without a colorbar. The text display method has been changed so that the full range of the colormap is used. The default size is also now set based on the number of classes. Added support for merging the meta features with the original input features in StackingRegressor (via use_features_in_secondary ) like it is already supported in the other Stacking classes. ( #418 ) Added a support_only to the association_rules function, which allow constructing association rules (based on the support metric only) for cropped input DataFrames that don't contain a complete set of antecedent and consequent support values. ( #421 )","title":"New Features"},{"location":"CHANGELOG/#changes_16","text":"Itemsets generated with apriori are now frozenset s ( #393 by William Laney and #394 ) Now raises an error if a input DataFrame to apriori contains non 0, 1, True, False values. #419 )","title":"Changes"},{"location":"CHANGELOG/#bug-fixes_11","text":"Allow mlxtend estimators to be cloned via scikit-learn's clone function. ( #374 ) Fixes bug to allow the correct use of refit=False in StackingRegressor and StackingCVRegressor ( #384 and ( #385 ) by selay01 ) Allow StackingClassifier to work with sparse matrices when use_features_in_secondary=True ( #408 by Floris Hoogenbook ) Allow StackingCVRegressor to work with sparse matrices when use_features_in_secondary=True ( #416 ) Allow StackingCVClassifier to work with sparse matrices when use_features_in_secondary=True ( #417 )","title":"Bug Fixes"},{"location":"CHANGELOG/#version-0120-2018-21-04","text":"","title":"Version 0.12.0 (2018-21-04)"},{"location":"CHANGELOG/#downloads_17","text":"Source code (zip) Source code (tar.gz)","title":"Downloads"},{"location":"CHANGELOG/#new-features_10","text":"A new feature_importance_permuation function to compute the feature importance in classifiers and regressors via the permutation importance method ( #358 ) The fit method of the ExhaustiveFeatureSelector now optionally accepts **fit_params for the estimator that is used for the feature selection. ( #354 by Zach Griffith) The fit method of the SequentialFeatureSelector now optionally accepts **fit_params for the estimator that is used for the feature selection. ( #350 by Zach Griffith)","title":"New Features"},{"location":"CHANGELOG/#changes_17","text":"Replaced plot_decision_regions colors by a colorblind-friendly palette and adds contour lines for decision regions. ( #348 ) All stacking estimators now raise NonFittedErrors if any method for inference is called prior to fitting the estimator. ( #353 ) Renamed the refit parameter of both the StackingClassifier and StackingCVClassifier to use_clones to be more explicit and less misleading. ( #368 )","title":"Changes"},{"location":"CHANGELOG/#bug-fixes_12","text":"Various changes in the documentation and documentation tools to fix formatting issues ( #363 ) Fixes a bug where the StackingCVClassifier 's meta features were not stored in the original order when shuffle=True ( #370 ) Many documentation improvements, including links to the User Guides in the API docs ( #371 )","title":"Bug Fixes"},{"location":"CHANGELOG/#version-0110-2018-03-14","text":"","title":"Version 0.11.0 (2018-03-14)"},{"location":"CHANGELOG/#downloads_18","text":"Source code (zip) Source code (tar.gz)","title":"Downloads"},{"location":"CHANGELOG/#new-features_11","text":"New function implementing the resampled paired t-test procedure ( paired_ttest_resampled ) to compare the performance of two models. ( #323 ) New function implementing the k-fold paired t-test procedure ( paired_ttest_kfold_cv ) to compare the performance of two models (also called k-hold-out paired t-test). ( #324 ) New function implementing the 5x2cv paired t-test procedure ( paired_ttest_5x2cv ) proposed by Dieterrich (1998) to compare the performance of two models. ( #325 ) A refit parameter was added to stacking classes (similar to the refit parameter in the EnsembleVoteClassifier ), to support classifiers and regressors that follow the scikit-learn API but are not compatible with scikit-learn's clone function. ( #322 ) The ColumnSelector now has a drop_axis argument to use it in pipelines with CountVectorizers . ( #333 )","title":"New Features"},{"location":"CHANGELOG/#changes_18","text":"Raises an informative error message if predict or predict_meta_features is called prior to calling the fit method in StackingRegressor and StackingCVRegressor . ( #315 ) The plot_decision_regions function now automatically determines the optimal setting based on the feature dimensions and supports anti-aliasing. The old res parameter has been deprecated. ( #309 by Guillaume Poirier-Morency ) Apriori code is faster due to optimization in onehot transformation and the amount of candidates generated by the apriori algorithm. ( #327 by Jakub Smid ) The OnehotTransactions class (which is typically often used in combination with the apriori function for association rule mining) is now more memory efficient as it uses boolean arrays instead of integer arrays. In addition, the OnehotTransactions class can be now be provided with sparse argument to generate sparse representations of the onehot matrix to further improve memory efficiency. ( #328 by Jakub Smid ) The OneHotTransactions has been deprecated and replaced by the TransactionEncoder . ( #332 The plot_decision_regions function now has three new parameters, scatter_kwargs , contourf_kwargs , and scatter_highlight_kwargs , that can be used to modify the plotting style. ( #342 by James Bourbeau )","title":"Changes"},{"location":"CHANGELOG/#bug-fixes_13","text":"Fixed issue when class labels were provided to the EnsembleVoteClassifier when refit was set to false . ( #322 ) Allow arrays with 16-bit and 32-bit precision in plot_decision_regions function. ( #337 ) Fixed bug that raised an indexing error if the number of items was <= 1 when computing association rules using the conviction metric. ( #340 )","title":"Bug Fixes"},{"location":"CHANGELOG/#version-0100-2017-12-22","text":"","title":"Version 0.10.0 (2017-12-22)"},{"location":"CHANGELOG/#downloads_19","text":"Source code (zip) Source code (tar.gz)","title":"Downloads"},{"location":"CHANGELOG/#new-features_12","text":"New store_train_meta_features parameter for fit in StackingCVRegressor. if True, train meta-features are stored in self.train_meta_features_ . New pred_meta_features method for StackingCVRegressor . People can get test meta-features using this method. ( #294 via takashioya ) The new store_train_meta_features attribute and pred_meta_features method for the StackingCVRegressor were also added to the StackingRegressor , StackingClassifier , and StackingCVClassifier ( #299 & #300 ) New function ( evaluate.mcnemar_tables ) for creating multiple 2x2 contigency from model predictions arrays that can be used in multiple McNemar (post-hoc) tests or Cochran's Q or F tests, etc. ( #307 ) New function ( evaluate.cochrans_q ) for performing Cochran's Q test to compare the accuracy of multiple classifiers. ( #310 )","title":"New Features"},{"location":"CHANGELOG/#changes_19","text":"Added requirements.txt to setup.py . ( #304 via Colin Carrol )","title":"Changes"},{"location":"CHANGELOG/#bug-fixes_14","text":"Improved numerical stability for p-values computed via the the exact McNemar test ( #306 ) nose is not required to use the library ( #302 )","title":"Bug Fixes"},{"location":"CHANGELOG/#version-091-2017-11-19","text":"","title":"Version 0.9.1 (2017-11-19)"},{"location":"CHANGELOG/#downloads_20","text":"Source code (zip) Source code (tar.gz)","title":"Downloads"},{"location":"CHANGELOG/#new-features_13","text":"Added mlxtend.evaluate.bootstrap_point632_score to evaluate the performance of estimators using the .632 bootstrap. ( #283 ) New max_len parameter for the frequent itemset generation via the apriori function to allow for early stopping. ( #270 )","title":"New Features"},{"location":"CHANGELOG/#changes_20","text":"All feature index tuples in SequentialFeatureSelector or now in sorted order. ( #262 ) The SequentialFeatureSelector now runs the continuation of the floating inclusion/exclusion as described in Novovicova & Kittler (1994). Note that this didn't cause any difference in performance on any of the test scenarios but could lead to better performance in certain edge cases. ( #262 ) utils.Counter now accepts a name variable to help distinguish between multiple counters, time precision can be set with the 'precision' kwarg and the new attribute end_time holds the time the last iteration completed. ( #278 via Mathew Savage )","title":"Changes"},{"location":"CHANGELOG/#bug-fixes_15","text":"Fixed an deprecation error that occured with McNemar test when using SciPy 1.0. ( #283 )","title":"Bug Fixes"},{"location":"CHANGELOG/#version-090-2017-10-21","text":"","title":"Version 0.9.0 (2017-10-21)"},{"location":"CHANGELOG/#downloads_21","text":"Source code (zip) Source code (tar.gz)","title":"Downloads"},{"location":"CHANGELOG/#new-features_14","text":"Added evaluate.permutation_test , a permutation test for hypothesis testing (or A/B testing) to test if two samples come from the same distribution. Or in other words, a procedure to test the null hypothesis that that two groups are not significantly different (e.g., a treatment and a control group). ( #250 ) Added 'leverage' and 'conviction as evaluation metrics to the frequent_patterns.association_rules function. ( #246 & #247 ) Added a loadings_ attribute to PrincipalComponentAnalysis to compute the factor loadings of the features on the principal components. ( #251 ) Allow grid search over classifiers/regressors in ensemble and stacking estimators. ( #259 ) New make_multiplexer_dataset function that creates a dataset generated by a n-bit Boolean multiplexer for evaluating supervised learning algorithms. ( #263 ) Added a new BootstrapOutOfBag class, an implementation of the out-of-bag bootstrap to evaluate supervised learning algorithms. ( #265 ) The parameters for StackingClassifier , StackingCVClassifier , StackingRegressor , StackingCVRegressor , and EnsembleVoteClassifier can now be tuned using scikit-learn's GridSearchCV ( #254 via James Bourbeau )","title":"New Features"},{"location":"CHANGELOG/#changes_21","text":"The 'support' column returned by frequent_patterns.association_rules was changed to compute the support of \"antecedant union consequent\", and new antecedant support' and 'consequent support' column were added to avoid ambiguity. ( #245 ) Allow the OnehotTransactions to be cloned via scikit-learn's clone function, which is required by e.g., scikit-learn's FeatureUnion or GridSearchCV (via Iaroslav Shcherbatyi ). ( #249 )","title":"Changes"},{"location":"CHANGELOG/#bug-fixes_16","text":"Fix issues with self._init_time parameter in _IterativeModel subclasses. ( #256 ) Fix imprecision bug that occurred in plot_ecdf when run on Python 2.7. ( 264 ) The vectors from SVD in PrincipalComponentAnalysis are now being scaled so that the eigenvalues via solver='eigen' and solver='svd' now store eigenvalues that have the same magnitudes. ( #251 )","title":"Bug Fixes"},{"location":"CHANGELOG/#version-080-2017-09-09","text":"","title":"Version 0.8.0 (2017-09-09)"},{"location":"CHANGELOG/#downloads_22","text":"Source code (zip) Source code (tar.gz)","title":"Downloads"},{"location":"CHANGELOG/#new-features_15","text":"Added a mlxtend.evaluate.bootstrap that implements the ordinary nonparametric bootstrap to bootstrap a single statistic (for example, the mean. median, R^2 of a regression fit, and so forth) #232 SequentialFeatureSelecor 's k_features now accepts a string argument \"best\" or \"parsimonious\" for more \"automated\" feature selection. For instance, if \"best\" is provided, the feature selector will return the feature subset with the best cross-validation performance. If \"parsimonious\" is provided as an argument, the smallest feature subset that is within one standard error of the cross-validation performance will be selected. #238","title":"New Features"},{"location":"CHANGELOG/#changes_22","text":"SequentialFeatureSelector now uses np.nanmean over normal mean to support scorers that may return np.nan #211 (via mrkaiser ) The skip_if_stuck parameter was removed from SequentialFeatureSelector in favor of a more efficient implementation comparing the conditional inclusion/exclusion results (in the floating versions) to the performances of previously sampled feature sets that were cached #237 ExhaustiveFeatureSelector was modified to consume substantially less memory #195 (via Adam Erickson )","title":"Changes"},{"location":"CHANGELOG/#bug-fixes_17","text":"Fixed a bug where the SequentialFeatureSelector selected a feature subset larger than then specified via the k_features tuple max-value #213","title":"Bug Fixes"},{"location":"CHANGELOG/#version-070-2017-06-22","text":"","title":"Version 0.7.0 (2017-06-22)"},{"location":"CHANGELOG/#downloads_23","text":"Source code (zip) Source code (tar.gz)","title":"Downloads"},{"location":"CHANGELOG/#new-features_16","text":"New mlxtend.plotting.ecdf function for plotting empirical cumulative distribution functions ( #196 ). New StackingCVRegressor for stacking regressors with out-of-fold predictions to prevent overfitting ( #201 via Eike Dehling ).","title":"New Features"},{"location":"CHANGELOG/#changes_23","text":"The TensorFlow estimator have been removed from mlxtend, since TensorFlow has now very convenient ways to build on estimators, which render those implementations obsolete. plot_decision_regions now supports plotting decision regions for more than 2 training features #189 , via James Bourbeau ). Parallel execution in mlxtend.feature_selection.SequentialFeatureSelector and mlxtend.feature_selection.ExhaustiveFeatureSelector is now performed over different feature subsets instead of the different cross-validation folds to better utilize machines with multiple processors if the number of features is large ( #193 , via @whalebot-helmsman ). Raise meaningful error messages if pandas DataFrame s or Python lists of lists are fed into the StackingCVClassifer as a fit arguments ( 198 ). The n_folds parameter of the StackingCVClassifier was changed to cv and can now accept any kind of cross validation technique that is available from scikit-learn. For example, StackingCVClassifier(..., cv=StratifiedKFold(n_splits=3)) or StackingCVClassifier(..., cv=GroupKFold(n_splits=3)) ( #203 , via Konstantinos Paliouras ).","title":"Changes"},{"location":"CHANGELOG/#bug-fixes_18","text":"SequentialFeatureSelector now correctly accepts a None argument for the scoring parameter to infer the default scoring metric from scikit-learn classifiers and regressors ( #171 ). The plot_decision_regions function now supports pre-existing axes objects generated via matplotlib's plt.subplots . ( #184 , see example ) Made math.num_combinations and math.num_permutations numerically stable for large numbers of combinations and permutations ( #200 ).","title":"Bug Fixes"},{"location":"CHANGELOG/#version-060-2017-03-18","text":"","title":"Version 0.6.0 (2017-03-18)"},{"location":"CHANGELOG/#downloads_24","text":"Source code (zip) Source code (tar.gz)","title":"Downloads"},{"location":"CHANGELOG/#new-features_17","text":"An association_rules function is implemented that allows to generate rules based on a list of frequent itemsets (via Joshua Goerner ).","title":"New Features"},{"location":"CHANGELOG/#changes_24","text":"Adds a black edgecolor to plots via plotting.plot_decision_regions to make markers more distinguishable from the background in matplotlib>=2.0 . The association submodule was renamed to frequent_patterns .","title":"Changes"},{"location":"CHANGELOG/#bug-fixes_19","text":"The DataFrame index of apriori results are now unique and ordered. Fixed typos in autompg and wine datasets (via James Bourbeau ).","title":"Bug Fixes"},{"location":"CHANGELOG/#version-051-2017-02-14","text":"","title":"Version 0.5.1 (2017-02-14)"},{"location":"CHANGELOG/#downloads_25","text":"Source code (zip) Source code (tar.gz)","title":"Downloads"},{"location":"CHANGELOG/#new-features_18","text":"The EnsembleVoteClassifier has a new refit attribute that prevents refitting classifiers if refit=False to save computational time. Added a new lift_score function in evaluate to compute lift score (via Batuhan Bardak ). StackingClassifier and StackingRegressor support multivariate targets if the underlying models do (via kernc ). StackingClassifier has a new use_features_in_secondary attribute like StackingCVClassifier .","title":"New Features"},{"location":"CHANGELOG/#changes_25","text":"Changed default verbosity level in SequentialFeatureSelector to 0 The EnsembleVoteClassifier now raises a NotFittedError if the estimator wasn't fit before calling predict . (via Anton Loss ) Added new TensorFlow variable initialization syntax to guarantee compatibility with TensorFlow 1.0","title":"Changes"},{"location":"CHANGELOG/#bug-fixes_20","text":"Fixed wrong default value for k_features in SequentialFeatureSelector Cast selected feature subsets in the SequentialFeautureSelector as sets to prevent the iterator from getting stuck if the k_idx are different permutations of the same combination (via Zac Wellmer ). Fixed an issue with learning curves that caused the performance metrics to be reversed (via ipashchenko ) Fixed a bug that could occur in the SequentialFeatureSelector if there are similarly-well performing subsets in the floating variants (via Zac Wellmer ).","title":"Bug Fixes"},{"location":"CHANGELOG/#version-050-2016-11-09","text":"","title":"Version 0.5.0 (2016-11-09)"},{"location":"CHANGELOG/#downloads_26","text":"Source code (zip) Source code (tar.gz)","title":"Downloads"},{"location":"CHANGELOG/#new-features_19","text":"New ExhaustiveFeatureSelector estimator in mlxtend.feature_selection for evaluating all feature combinations in a specified range The StackingClassifier has a new parameter average_probas that is set to True by default to maintain the current behavior. A deprecation warning was added though, and it will default to False in future releases (0.6.0); average_probas=False will result in stacking of the level-1 predicted probabilities rather than averaging these. New StackingCVClassifier estimator in 'mlxtend.classifier' for implementing a stacking ensemble that uses cross-validation techniques for training the meta-estimator to avoid overfitting ( Reiichiro Nakano ) New OnehotTransactions encoder class added to the preprocessing submodule for transforming transaction data into a one-hot encoded array The SequentialFeatureSelector estimator in mlxtend.feature_selection now is safely stoppable mid-process by control+c, and deprecated print_progress in favor of a more tunable verbose parameter ( Will McGinnis ) New apriori function in association to extract frequent itemsets from transaction data for association rule mining New checkerboard_plot function in plotting to plot checkerboard tables / heat maps New mcnemar_table and mcnemar functions in evaluate to compute 2x2 contingency tables and McNemar's test","title":"New Features"},{"location":"CHANGELOG/#changes_26","text":"All plotting functions have been moved to mlxtend.plotting for compatibility reasons with continuous integration services and to make the installation of matplotlib optional for users of mlxtend 's core functionality Added a compatibility layer for scikit-learn 0.18 using the new model_selection module while maintaining backwards compatibility to scikit-learn 0.17.","title":"Changes"},{"location":"CHANGELOG/#bug-fixes_21","text":"mlxtend.plotting.plot_decision_regions now draws decision regions correctly if more than 4 class labels are present Raise AttributeError in plot_decision_regions when the X_higlight argument is a 1D array ( chkoar )","title":"Bug Fixes"},{"location":"CHANGELOG/#version-042-2016-08-24","text":"","title":"Version 0.4.2 (2016-08-24)"},{"location":"CHANGELOG/#downloads_27","text":"Source code (zip) Source code (tar.gz) PDF documentation","title":"Downloads"},{"location":"CHANGELOG/#new-features_20","text":"Added preprocessing.CopyTransformer , a mock class that returns copies of imput arrays via transform and fit_transform","title":"New Features"},{"location":"CHANGELOG/#changes_27","text":"Added AppVeyor to CI to ensure MS Windows compatibility Dataset are now saved as compressed .txt or .csv files rather than being imported as Python objects feature_selection.SequentialFeatureSelector now supports the selection of k_features using a tuple to specify a \"min-max\" k_features range Added \"SVD solver\" option to the PrincipalComponentAnalysis Raise a AttributeError with \"not fitted\" message in SequentialFeatureSelector if transform or get_metric_dict are called prior to fit Use small, positive bias units in TfMultiLayerPerceptron 's hidden layer(s) if the activations are ReLUs in order to avoid dead neurons Added an optional clone_estimator parameter to the SequentialFeatureSelector that defaults to True , avoiding the modification of the original estimator objects More rigorous type and shape checks in the evaluate.plot_decision_regions function DenseTransformer now doesn't raise and error if the input array is not sparse API clean-up using scikit-learn's BaseEstimator as parent class for feature_selection.ColumnSelector","title":"Changes"},{"location":"CHANGELOG/#bug-fixes_22","text":"Fixed a problem when a tuple-range was provided as argument to the SequentialFeatureSelector 's k_features parameter and the scoring metric was more negative than -1 (e.g., as in scikit-learn's MSE scoring function) (wahutch](https://github.com/wahutch)) Fixed an AttributeError issue when verbose > 1 in StackingClassifier Fixed a bug in classifier.SoftmaxRegression where the mean values of the offsets were used to update the bias units rather than their sum Fixed rare bug in MLP _layer_mapping functions that caused a swap between the random number generation seed when initializing weights and biases","title":"Bug Fixes"},{"location":"CHANGELOG/#version-041-2016-05-01","text":"","title":"Version 0.4.1 (2016-05-01)"},{"location":"CHANGELOG/#downloads_28","text":"Source code (zip) Source code (tar.gz) PDF documentation","title":"Downloads"},{"location":"CHANGELOG/#new-features_21","text":"New TensorFlow estimator for Linear Regression ( tf_regressor.TfLinearRegression ) New k-means clustering estimator ( cluster.Kmeans ) New TensorFlow k-means clustering estimator ( tf_cluster.Kmeans )","title":"New Features"},{"location":"CHANGELOG/#changes_28","text":"Due to refactoring of the estimator classes, the init_weights parameter of the fit methods was globally renamed to init_params Overall performance improvements of estimators due to code clean-up and refactoring Added several additional checks for correct array types and more meaningful exception messages Added optional dropout to the tf_classifier.TfMultiLayerPerceptron classifier for regularization Added an optional decay parameter to the tf_classifier.TfMultiLayerPerceptron classifier for adaptive learning via an exponential decay of the learning rate eta Replaced old NeuralNetMLP by more streamlined MultiLayerPerceptron ( classifier.MultiLayerPerceptron ); now also with softmax in the output layer and categorical cross-entropy loss. Unified init_params parameter for fit functions to continue training where the algorithm left off (if supported)","title":"Changes"},{"location":"CHANGELOG/#version-040-2016-04-09","text":"","title":"Version 0.4.0 (2016-04-09)"},{"location":"CHANGELOG/#new-features_22","text":"New TfSoftmaxRegression classifier using Tensorflow ( tf_classifier.TfSoftmaxRegression ) New SoftmaxRegression classifier ( classifier.SoftmaxRegression ) New TfMultiLayerPerceptron classifier using Tensorflow ( tf_classifier.TfMultiLayerPerceptron ) New StackingRegressor ( regressor.StackingRegressor ) New StackingClassifier ( classifier.StackingClassifier ) New function for one-hot encoding of class labels ( preprocessing.one_hot ) Added GridSearch support to the SequentialFeatureSelector ( feature_selection/.SequentialFeatureSelector ) evaluate.plot_decision_regions improvements: Function now handles class y-class labels correctly if array is of type float Correct handling of input arguments markers and colors Accept an existing Axes via the ax argument New print_progress parameter for all generalized models and multi-layer neural networks for printing time elapsed, ETA, and the current cost of the current epoch Minibatch learning for classifier.LogisticRegression , classifier.Adaline , and regressor.LinearRegression plus streamlined API New Principal Component Analysis class via mlxtend.feature_extraction.PrincipalComponentAnalysis New RBF Kernel Principal Component Analysis class via mlxtend.feature_extraction.RBFKernelPCA New Linear Discriminant Analysis class via mlxtend.feature_extraction.LinearDiscriminantAnalysis","title":"New Features"},{"location":"CHANGELOG/#changes_29","text":"The column parameter in mlxtend.preprocessing.standardize now defaults to None to standardize all columns more conveniently","title":"Changes"},{"location":"CHANGELOG/#version-030-2016-01-31","text":"","title":"Version 0.3.0 (2016-01-31)"},{"location":"CHANGELOG/#downloads_29","text":"Source code (zip) Source code (tar.gz)","title":"Downloads"},{"location":"CHANGELOG/#new-features_23","text":"Added a progress bar tracker to classifier.NeuralNetMLP Added a function to score predicted vs. target class labels evaluate.scoring Added confusion matrix functions to create ( evaluate.confusion_matrix ) and plot ( evaluate.plot_confusion_matrix ) confusion matrices New style parameter and improved axis scaling in mlxtend.evaluate.plot_learning_curves Added loadlocal_mnist to mlxtend.data for streaming MNIST from a local byte files into numpy arrays New NeuralNetMLP parameters: random_weights , shuffle_init , shuffle_epoch New SFS features such as the generation of pandas DataFrame results tables and plotting functions (with confidence intervals, standard deviation, and standard error bars) Added support for regression estimators in SFS Added Boston housing dataset New shuffle parameter for classifier.NeuralNetMLP","title":"New Features"},{"location":"CHANGELOG/#changes_30","text":"The mlxtend.preprocessing.standardize function now optionally returns the parameters, which are estimated from the array, for re-use. A further improvement makes the standardize function smarter in order to avoid zero-division errors Cosmetic improvements to the evaluate.plot_decision_regions function such as hiding plot axes Renaming of classifier.EnsembleClassfier to classifier.EnsembleVoteClassifier Improved random weight initialization in Perceptron , Adaline , LinearRegression , and LogisticRegression Changed learning parameter of mlxtend.classifier.Adaline to solver and added \"normal equation\" as closed-form solution solver Hide y-axis labels in mlxtend.evaluate.plot_decision_regions in 1 dimensional evaluations Sequential Feature Selection algorithms were unified into a single SequentialFeatureSelector class with parameters to enable floating selection and toggle between forward and backward selection. Stratified sampling of MNIST (now 500x random samples from each of the 10 digit categories) Renaming mlxtend.plotting to mlxtend.general_plotting in order to distinguish general plotting function from specialized utility function such as evaluate.plot_decision_regions","title":"Changes"},{"location":"CHANGELOG/#version-029-2015-07-14","text":"","title":"Version 0.2.9 (2015-07-14)"},{"location":"CHANGELOG/#downloads_30","text":"Source code (zip) Source code (tar.gz)","title":"Downloads"},{"location":"CHANGELOG/#new-features_24","text":"Sequential Feature Selection algorithms: SFS, SFFS, SBS, and SFBS","title":"New Features"},{"location":"CHANGELOG/#changes_31","text":"Changed regularization & lambda parameters in LogisticRegression to single parameter l2_lambda","title":"Changes"},{"location":"CHANGELOG/#version-028-2015-06-27","text":"API changes: mlxtend.sklearn.EnsembleClassifier -> mlxtend.classifier.EnsembleClassifier mlxtend.sklearn.ColumnSelector -> mlxtend.feature_selection.ColumnSelector mlxtend.sklearn.DenseTransformer -> mlxtend.preprocessing.DenseTransformer mlxtend.pandas.standardizing -> mlxtend.preprocessing.standardizing mlxtend.pandas.minmax_scaling -> mlxtend.preprocessing.minmax_scaling mlxtend.matplotlib -> mlxtend.plotting Added momentum learning parameter (alpha coefficient) to mlxtend.classifier.NeuralNetMLP . Added adaptive learning rate (decrease constant) to mlxtend.classifier.NeuralNetMLP . mlxtend.pandas.minmax_scaling became mlxtend.preprocessing.minmax_scaling and also supports NumPy arrays now mlxtend.pandas.standardizing became mlxtend.preprocessing.standardizing and now supports both NumPy arrays and pandas DataFrames; also, now ddof parameters to set the degrees of freedom when calculating the standard deviation","title":"Version 0.2.8 (2015-06-27)"},{"location":"CHANGELOG/#version-027-2015-06-20","text":"Added multilayer perceptron (feedforward artificial neural network) classifier as mlxtend.classifier.NeuralNetMLP . Added 5000 labeled trainingsamples from the MNIST handwritten digits dataset to mlxtend.data","title":"Version 0.2.7 (2015-06-20)"},{"location":"CHANGELOG/#version-026-2015-05-08","text":"Added ordinary least square regression using different solvers (gradient and stochastic gradient descent, and the closed form solution (normal equation) Added option for random weight initialization to logistic regression classifier and updated l2 regularization Added wine dataset to mlxtend.data Added invert_axes parameter mlxtend.matplotlib.enrichtment_plot to optionally plot the \"Count\" on the x-axis New verbose parameter for mlxtend.sklearn.EnsembleClassifier by Alejandro C. Bahnsen Added mlxtend.pandas.standardizing to standardize columns in a Pandas DataFrame Added parameters linestyles and markers to mlxtend.matplotlib.enrichment_plot mlxtend.regression.lin_regplot automatically adds np.newaxis and works w. python lists Added tokenizers: mlxtend.text.extract_emoticons and mlxtend.text.extract_words_and_emoticons","title":"Version 0.2.6 (2015-05-08)"},{"location":"CHANGELOG/#version-025-2015-04-17","text":"Added Sequential Backward Selection (mlxtend.sklearn.SBS) Added X_highlight parameter to mlxtend.evaluate.plot_decision_regions for highlighting test data points. Added mlxtend.regression.lin_regplot to plot the fitted line from linear regression. Added mlxtend.matplotlib.stacked_barplot to conveniently produce stacked barplots using pandas DataFrame s. Added mlxtend.matplotlib.enrichment_plot","title":"Version 0.2.5 (2015-04-17)"},{"location":"CHANGELOG/#version-024-2015-03-15","text":"Added scoring to mlxtend.evaluate.learning_curves (by user pfsq) Fixed setup.py bug caused by the missing README.html file matplotlib.category_scatter for pandas DataFrames and Numpy arrays","title":"Version 0.2.4 (2015-03-15)"},{"location":"CHANGELOG/#version-023-2015-03-11","text":"Added Logistic regression Gradient descent and stochastic gradient descent perceptron was changed to Adaline (Adaptive Linear Neuron) Perceptron and Adaline for {0, 1} classes Added mlxtend.preprocessing.shuffle_arrays_unison function to shuffle one or more NumPy arrays. Added shuffle and random seed parameter to stochastic gradient descent classifier. Added rstrip parameter to mlxtend.file_io.find_filegroups to allow trimming of base names. Added ignore_substring parameter to mlxtend.file_io.find_filegroups and find_files . Replaced .rstrip in mlxtend.file_io.find_filegroups with more robust regex. Gridsearch support for mlxtend.sklearn.EnsembleClassifier","title":"Version 0.2.3 (2015-03-11)"},{"location":"CHANGELOG/#version-022-2015-03-01","text":"Improved robustness of EnsembleClassifier. Extended plot_decision_regions() functionality for plotting 1D decision boundaries. Function matplotlib.plot_decision_regions was reorganized to evaluate.plot_decision_regions . evaluate.plot_learning_curves() function added. Added Rosenblatt, gradient descent, and stochastic gradient descent perceptrons.","title":"Version 0.2.2 (2015-03-01)"},{"location":"CHANGELOG/#version-021-2015-01-20","text":"Added mlxtend.pandas.minmax_scaling - a function to rescale pandas DataFrame columns. Slight update to the EnsembleClassifier interface (additional voting parameter) Fixed EnsembleClassifier to return correct class labels if class labels are not integers from 0 to n. Added new matplotlib function to plot decision regions of classifiers.","title":"Version 0.2.1 (2015-01-20)"},{"location":"CHANGELOG/#version-020-2015-01-13","text":"Improved mlxtend.text.generalize_duplcheck to remove duplicates and prevent endless looping issue. Added recursive search parameter to mlxtend.file_io.find_files. Added check_ext parameter mlxtend.file_io.find_files to search based on file extensions. Default parameter to ignore invisible files for mlxtend.file_io.find. Added transform and fit_transform to the EnsembleClassifier . Added mlxtend.file_io.find_filegroups function.","title":"Version 0.2.0 (2015-01-13)"},{"location":"CHANGELOG/#version-019-2015-01-10","text":"Implemented scikit-learn EnsembleClassifier (majority voting rule) class.","title":"Version 0.1.9 (2015-01-10)"},{"location":"CHANGELOG/#version-018-2015-01-07","text":"Improvements to mlxtend.text.generalize_names to handle certain Dutch last name prefixes (van, van der, de, etc.). Added mlxtend.text.generalize_name_duplcheck function to apply mlxtend.text.generalize_names function to a pandas DataFrame without creating duplicates.","title":"Version 0.1.8 (2015-01-07)"},{"location":"CHANGELOG/#version-017-2015-01-07","text":"Added text utilities with name generalization function. Added and file_io utilities.","title":"Version 0.1.7 (2015-01-07)"},{"location":"CHANGELOG/#version-016-2015-01-04","text":"Added combinations and permutations estimators.","title":"Version 0.1.6 (2015-01-04)"},{"location":"CHANGELOG/#version-015-2014-12-11","text":"Added DenseTransformer for pipelines and grid search.","title":"Version 0.1.5 (2014-12-11)"},{"location":"CHANGELOG/#version-014-2014-08-20","text":"mean_centering function is now a Class that creates MeanCenterer objects that can be used to fit data via the fit method, and center data at the column means via the transform and fit_transform method.","title":"Version 0.1.4 (2014-08-20)"},{"location":"CHANGELOG/#version-013-2014-08-19","text":"Added preprocessing module and mean_centering function.","title":"Version 0.1.3 (2014-08-19)"},{"location":"CHANGELOG/#version-012-2014-08-19","text":"Added matplotlib utilities and remove_borders function.","title":"Version 0.1.2 (2014-08-19)"},{"location":"CHANGELOG/#version-011-2014-08-13","text":"Simplified code for ColumnSelector.","title":"Version 0.1.1 (2014-08-13)"},{"location":"CONTRIBUTING/","text":"How to Contribute I would be very happy about any kind of contributions that help to improve and extend the functionality of mlxtend. Quick Contributor Checklist This is a quick checklist about the different steps of a typical contribution to mlxtend (and other open source projects). Consider copying this list to a local text file (or the issue tracker) and checking off items as you go. 1) Making and testing code changes: [ ] Open a new \"issue\" on GitHub to discuss the new feature / bug fix [ ] Fork the mlxtend repository from GitHub (if not already done earlier) [ ] Create and check out a new topic branch (please don't make modifications in the master branch) [ ] Implement the new feature or apply the bug-fix [ ] Add appropriate unit test functions in mlxtend/*/tests [ ] Run PYTHONPATH='.' pytest ./mlxtend -sv and make sure that all unit tests pass [ ] Make sure the newly implemented feature has good test coverage: python -m pip install coverage # test all: # coverage run --source=mlxtend --branch -m pytest . coverage run --source=mlxtend --branch -m pytest mlxtend/<insert_path> coverage html [ ] Modify documentation in the appropriate location under mlxtend/docs/sources/ [ ] Add a note about the modification/contribution to the ./docs/sources/changelog.md file 2) Checking code style: When you check in a PR, mlxtend will run code style checks via flak8 and black. To make the contributor experience easier, we recommend you check the code style locally before pushing it to the repository. This way it is less likely that the automated checkers will complain and prompt you to make fixes. There are two ways you can do this: Option A : Running the tools manually [ ] Check for style issues by running flake8 ./mlxtend (you may want to run pytest again after you made modifications to the code) [ ] We recommend using black to format the code automatically according to recommended style changes. After installing black , you can do this via black [source_file_or_directory] [ ] Run isort which will sort the imports alphabetically. We recommend the following command: isort -p mlxtend --line-length 88 --multi-line 3 --profile black mypythonfile.py Option B : Using pre-commit hooks (recommended) The pre-commit hooks for mlxtend will check your code via flake8 , black , and isort automatically before you make a git commit . You can read more about pre-commit hooks here . [ ] Install the pre-commit package via pip install pre-commit . [ ] In the mlxtend folder, run pre-commit install (you only have to do it once). 3) Submitting your code [ ] Push the topic branch to the server and create a pull request. [ ] Check the automated tests passed. [ ] The automatic PEP8 / black integrations may prompt you to modify the code stylistically. It would be nice if you could apply the suggested changes. Tips for Contributors Getting Started - Creating a New Issue and Forking the Repository If you don't have a GitHub account, yet, please create one to contribute to this project. Please submit a ticket for your issue to discuss the fix or new feature before too much time and effort is spent for the implementation. Fork the mlxtend repository from the GitHub web interface. Clone the mlxtend repository to your local machine by executing git clone https://github.com/<your_username>/mlxtend.git Syncing an Existing Fork If you already forked mlxtend earlier, you can bring you \"Fork\" up to date with the master branch as follows: 1. Configuring a remote that points to the upstream repository on GitHub List the current configured remote repository of your fork by executing $ git remote -v If you see something like origin https://github.com/<your username>/mlxtend.git (fetch) origin https://github.com/<your username>/mlxtend.git (push) you need to specify a new remote upstream repository via $ git remote add upstream https://github.com/rasbt/mlxtend.git Now, verify the new upstream repository you've specified for your fork by executing $ git remote -v You should see following output if everything is configured correctly: origin https://github.com/<your username>/mlxtend.git (fetch) origin https://github.com/<your username>/mlxtend.git (push) upstream https://github.com/rasbt/mlxtend.git (fetch) upstream https://github.com/rasbt/mlxtend.git (push) 2. Syncing your Fork First, fetch the updates of the original project's master branch by executing: $ git fetch upstream You should see the following output remote: Counting objects: xx, done. remote: Compressing objects: 100% (xx/xx), done. remote: Total xx (delta xx), reused xx (delta x) Unpacking objects: 100% (xx/xx), done. From https://github.com/rasbt/mlxtend * [new branch] master -> upstream/master This means that the commits to the rasbt/mlxtend master branch are now stored in the local branch upstream/master . If you are not already on your local project's master branch, execute $ git checkout master Finally, merge the changes in upstream/master to your local master branch by executing $ git merge upstream/master which will give you an output that looks similar to Updating xxx...xxx Fast-forward SOME FILE1 | 12 +++++++ SOME FILE2 | 10 +++++++ 2 files changed, 22 insertions(+), *The Main Workflow - Making Changes in a New Topic Branch Listed below are the 9 typical steps of a contribution. 1. Discussing the Feature or Modification Before you start coding, please discuss the new feature, bugfix, or other modification to the project on the project's issue tracker . Before you open a \"new issue,\" please do a quick search to see if a similar issue has been submitted already. 2. Creating a new feature branch Please avoid working directly on the master branch but create a new feature branch: $ git branch <new_feature> Switch to the new feature branch by executing $ git checkout <new_feature> 3. Developing the new feature / bug fix Now it's time to modify existing code or to contribute new code to the project. 4. Testing your code Add the respective unit tests and check if they pass: $ PYTHONPATH='.' pytest ./mlxtend ---with-coverage 5. Documenting changes Please add an entry to the mlxtend/docs/sources/changelog.md file. If it is a new feature, it would also be nice if you could update the documentation in appropriate location in mlxtend/sources . 6. Committing changes When you are ready to commit the changes, please provide a meaningful commit message: $ git add <modifies_files> # or `git add .` $ git commit -m '<meaningful commit message>' 7. Optional: squashing commits If you made multiple smaller commits, it would be nice if you could group them into a larger, summarizing commit. First, list your recent commit via Note Due to the improved GitHub UI, this is no longer necessary/encouraged. $ git log which will list the commits from newest to oldest in the following format by default: commit 046e3af8a9127df8eac879454f029937c8a31c41 Author: rasbt <mail@sebastianraschka.com> Date: Tue Nov 24 03:46:37 2015 -0500 fixed setup.py commit c3c00f6ba0e8f48bbe1c9081b8ae3817e57ecc5c Author: rasbt <mail@sebastianraschka.com> Date: Tue Nov 24 03:04:39 2015 -0500 documented feature x commit d87934fe8726c46f0b166d6290a3bf38915d6e75 Author: rasbt <mail@sebastianraschka.com> Date: Tue Nov 24 02:44:45 2015 -0500 added support for feature x Assuming that it would make sense to group these 3 commits into one, we can execute $ git rebase -i HEAD~3 which will bring our default git editor with the following contents: pick d87934f added support for feature x pick c3c00f6 documented feature x pick 046e3af fixed setup.py Since c3c00f6 and 046e3af are related to the original commit of feature x , let's keep the d87934f and squash the 2 following commits into this initial one by changes the lines to pick d87934f added support for feature x squash c3c00f6 documented feature x squash 046e3af fixed setup.py Now, save the changes in your editor. Now, quitting the editor will apply the rebase changes, and the editor will open a second time, prompting you to enter a new commit message. In this case, we could enter support for feature x to summarize the contributions. 8. Uploading changes Push your changes to a topic branch to the git server by executing: $ git push origin <feature_branch> 9. Submitting a pull request Go to your GitHub repository online, select the new feature branch, and submit a new pull request: Notes for Developers Building the documentation The documentation is built via MkDocs ; to ensure that the documentation is rendered correctly, you can view the documentation locally by executing mkdocs serve from the mlxtend/docs directory. For example, ~/github/mlxtend/docs$ mkdocs serve 1. Building the API documentation To build the API documentation, navigate to mlxtend/docs and execute the make_api.py file from this directory via ~/github/mlxtend/docs$ python make_api.py This should place the API documentation into the correct directories into the two directories: mlxtend/docs/sources/api_modules mlxtend/docs/sources/api_subpackes 2. Editing the User Guide The documents containing code examples for the \"User Guide\" are generated from IPython Notebook files. In order to convert a IPython notebook file to markdown after editing, please follow the following steps: Modify or edit the existing notebook. Execute all cells in the current notebook and make sure that no errors occur. Convert the notebook to markdown using the ipynb2markdown.py converter ~/github/mlxtend/docs$ python ipynb2markdown.py --ipynb ./sources/user_guide/subpackage/notebookname.ipynb Note If you are adding a new document, please also include it in the pages section in the mlxtend/docs/mkdocs.yml file. 3. Building static HTML files of the documentation First, please check the documenation via localhost (https://127.0.0.1:8000/): ~/github/mlxtend/docs$ mkdocs serve Next, build the static HTML files of the mlxtend documentation via ~/github/mlxtend/docs$ mkdocs build --clean To deploy the documentation, execute ~/github/mlxtend/docs$ mkdocs gh-deploy --clean 4. Generate a PDF of the documentation To generate a PDF version of the documentation, simply cd into the mlxtend/docs directory and execute: python md2pdf.py Uploading a new version to PyPI 1. Creating a new testing environment Assuming we are using conda , create a new python environment via $ conda create -n 'mlxtend-testing' python=3 numpy scipy pandas Next, activate the environment by executing $ source activate mlxtend-testing 2. Installing the package from local files Test the installation by executing the following from within the mlxtend main directory: $ pip install . -e the --record files.txt flag will create a files.txt file listing the locations where these files will be installed. Try to import the package to see if it works, for example, by executing $ python -c 'import mlxtend; print(mlxtend.__file__)' If everything seems to be fine, remove the installation via $ cat files.txt | xargs rm -rf ; rm files.txt Next, test if pip is able to install the packages. First, navigate to a different directory, and from there, install the package: $ pip uninstall $ pip install mlxtend and uninstall it again $ pip uninstall mlxtend 3. Deploying the package Consider deploying the package to the PyPI test server first. The setup instructions can be found here . First , install Twine if you don't have it already installed. E.g., use the following to install all recommended packages: $ python -m pip install twine build Second , create the distribution. This by default creates an sdist and wheel in the ./dist directory. $ python -m build Install the wheel and sdist to make sure they work. The distributions file names will change with each version. python -m pip install ./dist/mlxtend-0.23.0.dev0.tar.gz --force-reinstall python -m pip install ./dist/mlxtend-0.23.0.dev0-py3-none-any.whl --force-reinstall python -m pip uninstall mlxtend Third , upload the packages to the test server: $ twine upload --repository-url https://upload.pypi.org/legacy dist/ Then, install it and see if it works: $ pip install -i https://testpypi.python.org/pypi mlxtend Next, uninstall it again as follows: $ pip uninstall mlxtend Fourth , after this dry-run succeeded, repeat this process using the \"real\" PyPI: $ python -m twine upload dist/* 4. Removing the virtual environment Finally, to cleanup our local drive, remove the virtual testing environment via $ conda remove --name 'mlxtend-testing' --all Note : if you get an error like HTTPError: 403 Forbidden from https://upload.pypi.org/legacy/ make sure you have an up to date version of twine installed (helped me to uninstall in conda and install via pip.) 5. Updating the conda-forge recipe Once a new version of mlxtend has been uploaded to PyPI, update the conda-forge build recipe at https://github.com/conda-forge/mlxtend-feedstock by changing the version number in the recipe/meta.yaml file appropriately.","title":"How To Contribute"},{"location":"CONTRIBUTING/#how-to-contribute","text":"I would be very happy about any kind of contributions that help to improve and extend the functionality of mlxtend.","title":"How to Contribute"},{"location":"CONTRIBUTING/#quick-contributor-checklist","text":"This is a quick checklist about the different steps of a typical contribution to mlxtend (and other open source projects). Consider copying this list to a local text file (or the issue tracker) and checking off items as you go.","title":"Quick Contributor Checklist"},{"location":"CONTRIBUTING/#1-making-and-testing-code-changes","text":"[ ] Open a new \"issue\" on GitHub to discuss the new feature / bug fix [ ] Fork the mlxtend repository from GitHub (if not already done earlier) [ ] Create and check out a new topic branch (please don't make modifications in the master branch) [ ] Implement the new feature or apply the bug-fix [ ] Add appropriate unit test functions in mlxtend/*/tests [ ] Run PYTHONPATH='.' pytest ./mlxtend -sv and make sure that all unit tests pass [ ] Make sure the newly implemented feature has good test coverage: python -m pip install coverage # test all: # coverage run --source=mlxtend --branch -m pytest . coverage run --source=mlxtend --branch -m pytest mlxtend/<insert_path> coverage html [ ] Modify documentation in the appropriate location under mlxtend/docs/sources/ [ ] Add a note about the modification/contribution to the ./docs/sources/changelog.md file","title":"1) Making and testing code changes:"},{"location":"CONTRIBUTING/#2-checking-code-style","text":"When you check in a PR, mlxtend will run code style checks via flak8 and black. To make the contributor experience easier, we recommend you check the code style locally before pushing it to the repository. This way it is less likely that the automated checkers will complain and prompt you to make fixes. There are two ways you can do this: Option A : Running the tools manually [ ] Check for style issues by running flake8 ./mlxtend (you may want to run pytest again after you made modifications to the code) [ ] We recommend using black to format the code automatically according to recommended style changes. After installing black , you can do this via black [source_file_or_directory] [ ] Run isort which will sort the imports alphabetically. We recommend the following command: isort -p mlxtend --line-length 88 --multi-line 3 --profile black mypythonfile.py Option B : Using pre-commit hooks (recommended) The pre-commit hooks for mlxtend will check your code via flake8 , black , and isort automatically before you make a git commit . You can read more about pre-commit hooks here . [ ] Install the pre-commit package via pip install pre-commit . [ ] In the mlxtend folder, run pre-commit install (you only have to do it once).","title":"2) Checking code style:"},{"location":"CONTRIBUTING/#3-submitting-your-code","text":"[ ] Push the topic branch to the server and create a pull request. [ ] Check the automated tests passed. [ ] The automatic PEP8 / black integrations may prompt you to modify the code stylistically. It would be nice if you could apply the suggested changes.","title":"3) Submitting your code"},{"location":"CONTRIBUTING/#tips-for-contributors","text":"","title":"Tips for Contributors"},{"location":"CONTRIBUTING/#getting-started-creating-a-new-issue-and-forking-the-repository","text":"If you don't have a GitHub account, yet, please create one to contribute to this project. Please submit a ticket for your issue to discuss the fix or new feature before too much time and effort is spent for the implementation. Fork the mlxtend repository from the GitHub web interface. Clone the mlxtend repository to your local machine by executing git clone https://github.com/<your_username>/mlxtend.git","title":"Getting Started - Creating a New Issue and Forking the Repository"},{"location":"CONTRIBUTING/#syncing-an-existing-fork","text":"If you already forked mlxtend earlier, you can bring you \"Fork\" up to date with the master branch as follows:","title":"Syncing an Existing Fork"},{"location":"CONTRIBUTING/#1-configuring-a-remote-that-points-to-the-upstream-repository-on-github","text":"List the current configured remote repository of your fork by executing $ git remote -v If you see something like origin https://github.com/<your username>/mlxtend.git (fetch) origin https://github.com/<your username>/mlxtend.git (push) you need to specify a new remote upstream repository via $ git remote add upstream https://github.com/rasbt/mlxtend.git Now, verify the new upstream repository you've specified for your fork by executing $ git remote -v You should see following output if everything is configured correctly: origin https://github.com/<your username>/mlxtend.git (fetch) origin https://github.com/<your username>/mlxtend.git (push) upstream https://github.com/rasbt/mlxtend.git (fetch) upstream https://github.com/rasbt/mlxtend.git (push)","title":"1. Configuring a remote that points to the upstream repository on GitHub"},{"location":"CONTRIBUTING/#2-syncing-your-fork","text":"First, fetch the updates of the original project's master branch by executing: $ git fetch upstream You should see the following output remote: Counting objects: xx, done. remote: Compressing objects: 100% (xx/xx), done. remote: Total xx (delta xx), reused xx (delta x) Unpacking objects: 100% (xx/xx), done. From https://github.com/rasbt/mlxtend * [new branch] master -> upstream/master This means that the commits to the rasbt/mlxtend master branch are now stored in the local branch upstream/master . If you are not already on your local project's master branch, execute $ git checkout master Finally, merge the changes in upstream/master to your local master branch by executing $ git merge upstream/master which will give you an output that looks similar to Updating xxx...xxx Fast-forward SOME FILE1 | 12 +++++++ SOME FILE2 | 10 +++++++ 2 files changed, 22 insertions(+),","title":"2. Syncing your Fork"},{"location":"CONTRIBUTING/#the-main-workflow-making-changes-in-a-new-topic-branch","text":"Listed below are the 9 typical steps of a contribution.","title":"*The Main Workflow - Making Changes in a New Topic Branch"},{"location":"CONTRIBUTING/#1-discussing-the-feature-or-modification","text":"Before you start coding, please discuss the new feature, bugfix, or other modification to the project on the project's issue tracker . Before you open a \"new issue,\" please do a quick search to see if a similar issue has been submitted already.","title":"1. Discussing the Feature or Modification"},{"location":"CONTRIBUTING/#2-creating-a-new-feature-branch","text":"Please avoid working directly on the master branch but create a new feature branch: $ git branch <new_feature> Switch to the new feature branch by executing $ git checkout <new_feature>","title":"2. Creating a new feature branch"},{"location":"CONTRIBUTING/#3-developing-the-new-feature-bug-fix","text":"Now it's time to modify existing code or to contribute new code to the project.","title":"3. Developing the new feature / bug fix"},{"location":"CONTRIBUTING/#4-testing-your-code","text":"Add the respective unit tests and check if they pass: $ PYTHONPATH='.' pytest ./mlxtend ---with-coverage","title":"4. Testing your code"},{"location":"CONTRIBUTING/#5-documenting-changes","text":"Please add an entry to the mlxtend/docs/sources/changelog.md file. If it is a new feature, it would also be nice if you could update the documentation in appropriate location in mlxtend/sources .","title":"5. Documenting changes"},{"location":"CONTRIBUTING/#6-committing-changes","text":"When you are ready to commit the changes, please provide a meaningful commit message: $ git add <modifies_files> # or `git add .` $ git commit -m '<meaningful commit message>'","title":"6. Committing changes"},{"location":"CONTRIBUTING/#7-optional-squashing-commits","text":"If you made multiple smaller commits, it would be nice if you could group them into a larger, summarizing commit. First, list your recent commit via Note Due to the improved GitHub UI, this is no longer necessary/encouraged. $ git log which will list the commits from newest to oldest in the following format by default: commit 046e3af8a9127df8eac879454f029937c8a31c41 Author: rasbt <mail@sebastianraschka.com> Date: Tue Nov 24 03:46:37 2015 -0500 fixed setup.py commit c3c00f6ba0e8f48bbe1c9081b8ae3817e57ecc5c Author: rasbt <mail@sebastianraschka.com> Date: Tue Nov 24 03:04:39 2015 -0500 documented feature x commit d87934fe8726c46f0b166d6290a3bf38915d6e75 Author: rasbt <mail@sebastianraschka.com> Date: Tue Nov 24 02:44:45 2015 -0500 added support for feature x Assuming that it would make sense to group these 3 commits into one, we can execute $ git rebase -i HEAD~3 which will bring our default git editor with the following contents: pick d87934f added support for feature x pick c3c00f6 documented feature x pick 046e3af fixed setup.py Since c3c00f6 and 046e3af are related to the original commit of feature x , let's keep the d87934f and squash the 2 following commits into this initial one by changes the lines to pick d87934f added support for feature x squash c3c00f6 documented feature x squash 046e3af fixed setup.py Now, save the changes in your editor. Now, quitting the editor will apply the rebase changes, and the editor will open a second time, prompting you to enter a new commit message. In this case, we could enter support for feature x to summarize the contributions.","title":"7. Optional: squashing commits"},{"location":"CONTRIBUTING/#8-uploading-changes","text":"Push your changes to a topic branch to the git server by executing: $ git push origin <feature_branch>","title":"8. Uploading changes"},{"location":"CONTRIBUTING/#9-submitting-a-pull-request","text":"Go to your GitHub repository online, select the new feature branch, and submit a new pull request:","title":"9. Submitting a pull request"},{"location":"CONTRIBUTING/#notes-for-developers","text":"","title":"Notes for Developers"},{"location":"CONTRIBUTING/#building-the-documentation","text":"The documentation is built via MkDocs ; to ensure that the documentation is rendered correctly, you can view the documentation locally by executing mkdocs serve from the mlxtend/docs directory. For example, ~/github/mlxtend/docs$ mkdocs serve","title":"Building the documentation"},{"location":"CONTRIBUTING/#1-building-the-api-documentation","text":"To build the API documentation, navigate to mlxtend/docs and execute the make_api.py file from this directory via ~/github/mlxtend/docs$ python make_api.py This should place the API documentation into the correct directories into the two directories: mlxtend/docs/sources/api_modules mlxtend/docs/sources/api_subpackes","title":"1. Building the API documentation"},{"location":"CONTRIBUTING/#2-editing-the-user-guide","text":"The documents containing code examples for the \"User Guide\" are generated from IPython Notebook files. In order to convert a IPython notebook file to markdown after editing, please follow the following steps: Modify or edit the existing notebook. Execute all cells in the current notebook and make sure that no errors occur. Convert the notebook to markdown using the ipynb2markdown.py converter ~/github/mlxtend/docs$ python ipynb2markdown.py --ipynb ./sources/user_guide/subpackage/notebookname.ipynb Note If you are adding a new document, please also include it in the pages section in the mlxtend/docs/mkdocs.yml file.","title":"2. Editing the User Guide"},{"location":"CONTRIBUTING/#3-building-static-html-files-of-the-documentation","text":"First, please check the documenation via localhost (https://127.0.0.1:8000/): ~/github/mlxtend/docs$ mkdocs serve Next, build the static HTML files of the mlxtend documentation via ~/github/mlxtend/docs$ mkdocs build --clean To deploy the documentation, execute ~/github/mlxtend/docs$ mkdocs gh-deploy --clean","title":"3. Building static HTML files of the documentation"},{"location":"CONTRIBUTING/#4-generate-a-pdf-of-the-documentation","text":"To generate a PDF version of the documentation, simply cd into the mlxtend/docs directory and execute: python md2pdf.py","title":"4. Generate a PDF of the documentation"},{"location":"CONTRIBUTING/#uploading-a-new-version-to-pypi","text":"","title":"Uploading a new version to PyPI"},{"location":"CONTRIBUTING/#1-creating-a-new-testing-environment","text":"Assuming we are using conda , create a new python environment via $ conda create -n 'mlxtend-testing' python=3 numpy scipy pandas Next, activate the environment by executing $ source activate mlxtend-testing","title":"1. Creating a new testing environment"},{"location":"CONTRIBUTING/#2-installing-the-package-from-local-files","text":"Test the installation by executing the following from within the mlxtend main directory: $ pip install . -e the --record files.txt flag will create a files.txt file listing the locations where these files will be installed. Try to import the package to see if it works, for example, by executing $ python -c 'import mlxtend; print(mlxtend.__file__)' If everything seems to be fine, remove the installation via $ cat files.txt | xargs rm -rf ; rm files.txt Next, test if pip is able to install the packages. First, navigate to a different directory, and from there, install the package: $ pip uninstall $ pip install mlxtend and uninstall it again $ pip uninstall mlxtend","title":"2. Installing the package from local files"},{"location":"CONTRIBUTING/#3-deploying-the-package","text":"Consider deploying the package to the PyPI test server first. The setup instructions can be found here . First , install Twine if you don't have it already installed. E.g., use the following to install all recommended packages: $ python -m pip install twine build Second , create the distribution. This by default creates an sdist and wheel in the ./dist directory. $ python -m build Install the wheel and sdist to make sure they work. The distributions file names will change with each version. python -m pip install ./dist/mlxtend-0.23.0.dev0.tar.gz --force-reinstall python -m pip install ./dist/mlxtend-0.23.0.dev0-py3-none-any.whl --force-reinstall python -m pip uninstall mlxtend Third , upload the packages to the test server: $ twine upload --repository-url https://upload.pypi.org/legacy dist/ Then, install it and see if it works: $ pip install -i https://testpypi.python.org/pypi mlxtend Next, uninstall it again as follows: $ pip uninstall mlxtend Fourth , after this dry-run succeeded, repeat this process using the \"real\" PyPI: $ python -m twine upload dist/*","title":"3. Deploying the package"},{"location":"CONTRIBUTING/#4-removing-the-virtual-environment","text":"Finally, to cleanup our local drive, remove the virtual testing environment via $ conda remove --name 'mlxtend-testing' --all Note : if you get an error like HTTPError: 403 Forbidden from https://upload.pypi.org/legacy/ make sure you have an up to date version of twine installed (helped me to uninstall in conda and install via pip.)","title":"4. Removing the virtual environment"},{"location":"CONTRIBUTING/#5-updating-the-conda-forge-recipe","text":"Once a new version of mlxtend has been uploaded to PyPI, update the conda-forge build recipe at https://github.com/conda-forge/mlxtend-feedstock by changing the version number in the recipe/meta.yaml file appropriately.","title":"5. Updating the conda-forge recipe"},{"location":"Code-of-Conduct/","text":"Contributor Covenant Code of Conduct Our Pledge In the interest of fostering an open and welcoming environment, we as contributors and maintainers pledge to make participation in our project and our community a harassment-free experience for everyone, regardless of age, body size, disability, ethnicity, sex characteristics, gender identity, and expression, level of experience, education, socio-economic status, nationality, personal appearance, race, religion, or sexual identity and orientation. Our Standards Examples of behavior that contributes to creating a positive environment include: Using welcoming and inclusive language; Being respectful of differing viewpoints and experiences; Gracefully accepting constructive criticism; Focusing on what is best for the community; Showing empathy towards other community members. Examples of unacceptable behavior by participants include: The use of sexualized language or imagery and unwelcome sexual attention or advances; Trolling, gaslighting, insulting/derogatory comments, and personal or political attacks; Public or private harassment; Publishing others' private information, such as a physical or electronic address, without explicit permission; Other conduct which could reasonably be considered inappropriate in a professional setting. Our Responsibilities Project maintainers are responsible for clarifying the standards of acceptable behavior and are expected to take appropriate and fair corrective action in response to any instances of unacceptable behavior. Project maintainers have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct, or to ban temporarily or permanently any contributor for other behaviors that they deem inappropriate, threatening, offensive, or harmful. Scope This Code of Conduct applies both within the project and public spaces when an individual represents the project or its community. Examples of representing a project or community include using an official project e-mail address, posting via an official social media account or acting as an appointed representative at an online or offline event. Representation of a project may be further defined and clarified by project maintainers. Enforcement Instances of abusive, harassing, or other unacceptable behavior may be reported by contacting the project team at mail@sebastianraschka.com. All complaints will be reviewed and investigated and will result in a response that is deemed necessary and appropriate to the circumstances. The project team is obligated to maintain confidentiality concerning the reporter of an incident. Further details of specific enforcement policies may be posted separately. Project maintainers who do not follow or enforce the Code of Conduct in good faith may face temporary or permanent repercussions as determined by other members of the project's leadership. Attribution This Code of Conduct is adapted from the Contributor Covenant , version 1.4, available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html For answers to common questions about this code of conduct, see https://www.contributor-covenant.org/faq","title":"Code of Conduct"},{"location":"Code-of-Conduct/#contributor-covenant-code-of-conduct","text":"","title":"Contributor Covenant Code of Conduct"},{"location":"Code-of-Conduct/#our-pledge","text":"In the interest of fostering an open and welcoming environment, we as contributors and maintainers pledge to make participation in our project and our community a harassment-free experience for everyone, regardless of age, body size, disability, ethnicity, sex characteristics, gender identity, and expression, level of experience, education, socio-economic status, nationality, personal appearance, race, religion, or sexual identity and orientation.","title":"Our Pledge"},{"location":"Code-of-Conduct/#our-standards","text":"Examples of behavior that contributes to creating a positive environment include: Using welcoming and inclusive language; Being respectful of differing viewpoints and experiences; Gracefully accepting constructive criticism; Focusing on what is best for the community; Showing empathy towards other community members. Examples of unacceptable behavior by participants include: The use of sexualized language or imagery and unwelcome sexual attention or advances; Trolling, gaslighting, insulting/derogatory comments, and personal or political attacks; Public or private harassment; Publishing others' private information, such as a physical or electronic address, without explicit permission; Other conduct which could reasonably be considered inappropriate in a professional setting.","title":"Our Standards"},{"location":"Code-of-Conduct/#our-responsibilities","text":"Project maintainers are responsible for clarifying the standards of acceptable behavior and are expected to take appropriate and fair corrective action in response to any instances of unacceptable behavior. Project maintainers have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct, or to ban temporarily or permanently any contributor for other behaviors that they deem inappropriate, threatening, offensive, or harmful.","title":"Our Responsibilities"},{"location":"Code-of-Conduct/#scope","text":"This Code of Conduct applies both within the project and public spaces when an individual represents the project or its community. Examples of representing a project or community include using an official project e-mail address, posting via an official social media account or acting as an appointed representative at an online or offline event. Representation of a project may be further defined and clarified by project maintainers.","title":"Scope"},{"location":"Code-of-Conduct/#enforcement","text":"Instances of abusive, harassing, or other unacceptable behavior may be reported by contacting the project team at mail@sebastianraschka.com. All complaints will be reviewed and investigated and will result in a response that is deemed necessary and appropriate to the circumstances. The project team is obligated to maintain confidentiality concerning the reporter of an incident. Further details of specific enforcement policies may be posted separately. Project maintainers who do not follow or enforce the Code of Conduct in good faith may face temporary or permanent repercussions as determined by other members of the project's leadership.","title":"Enforcement"},{"location":"Code-of-Conduct/#attribution","text":"This Code of Conduct is adapted from the Contributor Covenant , version 1.4, available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html For answers to common questions about this code of conduct, see https://www.contributor-covenant.org/faq","title":"Attribution"},{"location":"USER-GUIDE-INDEX/","text":"User Guide Index classifier Adaline EnsembleVoteClassifier LogisticRegression MultiLayerPerceptron OneRClassifier Perceptron SoftmaxRegression StackingClassifier StackingCVClassifier cluster Kmeans data autompg_data boston_housing_data iris_data loadlocal_mnist make_multiplexer_dataset mnist_data three_blobs_data wine_data evaluate accuracy_score bias_variance_decomp bootstrap bootstrap_point632_score BootstrapOutOfBag cochrans_q combined_ftest_5x2cv confusion_matrix create_counterfactual feature_importance_permutation ftest lift_score mcnemar_table mcnemar_tables mcnemar paired_ttest_5x2cv paired_ttest_kfold_cv paired_ttest_resampled permutation_test PredefinedHoldoutSplit proportion_difference RandomHoldoutSplit scoring feature_extraction LinearDiscriminantAnalysis PrincipalComponentAnalysis RBFKernelPCA feature_selection ColumnSelector ExhaustiveFeatureSelector SequentialFeatureSelector file_io find_filegroups find_files frequent_patterns apriori association_rules fpgrowth fpmax math num_combinations num_permutations plotting category_scatter checkerboard_plot ecdf enrichment_plot heatmap plot_confusion_matrix plot_pca_correlation_graph plot_decision_regions plot_learning_curves plot_linear_regression plot_sequential_feature_selection scatterplotmatrix stacked_barplot preprocessing CopyTransformer DenseTransformer MeanCenterer minmax_scaling one-hot_encoding shuffle_arrays_unison standardize TransactionEncoder regressor LinearRegression StackingCVRegressor StackingRegressor text generalize_names generalize_names_duplcheck tokenizer utils Counter","title":"User Guide Index"},{"location":"USER-GUIDE-INDEX/#user-guide-index","text":"","title":"User Guide Index"},{"location":"USER-GUIDE-INDEX/#classifier","text":"Adaline EnsembleVoteClassifier LogisticRegression MultiLayerPerceptron OneRClassifier Perceptron SoftmaxRegression StackingClassifier StackingCVClassifier","title":"classifier"},{"location":"USER-GUIDE-INDEX/#cluster","text":"Kmeans","title":"cluster"},{"location":"USER-GUIDE-INDEX/#data","text":"autompg_data boston_housing_data iris_data loadlocal_mnist make_multiplexer_dataset mnist_data three_blobs_data wine_data","title":"data"},{"location":"USER-GUIDE-INDEX/#evaluate","text":"accuracy_score bias_variance_decomp bootstrap bootstrap_point632_score BootstrapOutOfBag cochrans_q combined_ftest_5x2cv confusion_matrix create_counterfactual feature_importance_permutation ftest lift_score mcnemar_table mcnemar_tables mcnemar paired_ttest_5x2cv paired_ttest_kfold_cv paired_ttest_resampled permutation_test PredefinedHoldoutSplit proportion_difference RandomHoldoutSplit scoring","title":"evaluate"},{"location":"USER-GUIDE-INDEX/#feature_extraction","text":"LinearDiscriminantAnalysis PrincipalComponentAnalysis RBFKernelPCA","title":"feature_extraction"},{"location":"USER-GUIDE-INDEX/#feature_selection","text":"ColumnSelector ExhaustiveFeatureSelector SequentialFeatureSelector","title":"feature_selection"},{"location":"USER-GUIDE-INDEX/#file_io","text":"find_filegroups find_files","title":"file_io"},{"location":"USER-GUIDE-INDEX/#frequent_patterns","text":"apriori association_rules fpgrowth fpmax","title":"frequent_patterns"},{"location":"USER-GUIDE-INDEX/#math","text":"num_combinations num_permutations","title":"math"},{"location":"USER-GUIDE-INDEX/#plotting","text":"category_scatter checkerboard_plot ecdf enrichment_plot heatmap plot_confusion_matrix plot_pca_correlation_graph plot_decision_regions plot_learning_curves plot_linear_regression plot_sequential_feature_selection scatterplotmatrix stacked_barplot","title":"plotting"},{"location":"USER-GUIDE-INDEX/#preprocessing","text":"CopyTransformer DenseTransformer MeanCenterer minmax_scaling one-hot_encoding shuffle_arrays_unison standardize TransactionEncoder","title":"preprocessing"},{"location":"USER-GUIDE-INDEX/#regressor","text":"LinearRegression StackingCVRegressor StackingRegressor","title":"regressor"},{"location":"USER-GUIDE-INDEX/#text","text":"generalize_names generalize_names_duplcheck tokenizer","title":"text"},{"location":"USER-GUIDE-INDEX/#utils","text":"Counter","title":"utils"},{"location":"USER_GUIDE_INDEX/","text":"User Guide Index classifier Adaline EnsembleVoteClassifier LogisticRegression MultiLayerPerceptron OneRClassifier Perceptron SoftmaxRegression StackingClassifier StackingCVClassifier cluster Kmeans data autompg_data boston_housing_data iris_data loadlocal_mnist make_multiplexer_dataset mnist_data three_blobs_data wine_data evaluate accuracy_score bias_variance_decomp bootstrap bootstrap_point632_score BootstrapOutOfBag cochrans_q combined_ftest_5x2cv confusion_matrix create_counterfactual feature_importance_permutation ftest GroupTimeSeriesSplit lift_score mcnemar_table mcnemar_tables mcnemar paired_ttest_5x2cv paired_ttest_kfold_cv paired_ttest_resampled permutation_test PredefinedHoldoutSplit proportion_difference RandomHoldoutSplit scoring feature_extraction LinearDiscriminantAnalysis PrincipalComponentAnalysis RBFKernelPCA feature_selection ColumnSelector ExhaustiveFeatureSelector SequentialFeatureSelector file_io find_filegroups find_files frequent_patterns apriori association_rules fpgrowth fpmax math num_combinations num_permutations plotting category_scatter checkerboard_plot ecdf enrichment_plot heatmap plot_confusion_matrix plot_pca_correlation_graph plot_decision_regions plot_learning_curves plot_linear_regression plot_sequential_feature_selection scatterplotmatrix scatter_hist stacked_barplot preprocessing CopyTransformer DenseTransformer MeanCenterer minmax_scaling one-hot_encoding shuffle_arrays_unison standardize TransactionEncoder regressor LinearRegression StackingCVRegressor StackingRegressor text generalize_names generalize_names_duplcheck tokenizer utils Counter","title":"User Guide Index"},{"location":"USER_GUIDE_INDEX/#user-guide-index","text":"","title":"User Guide Index"},{"location":"USER_GUIDE_INDEX/#classifier","text":"Adaline EnsembleVoteClassifier LogisticRegression MultiLayerPerceptron OneRClassifier Perceptron SoftmaxRegression StackingClassifier StackingCVClassifier","title":"classifier"},{"location":"USER_GUIDE_INDEX/#cluster","text":"Kmeans","title":"cluster"},{"location":"USER_GUIDE_INDEX/#data","text":"autompg_data boston_housing_data iris_data loadlocal_mnist make_multiplexer_dataset mnist_data three_blobs_data wine_data","title":"data"},{"location":"USER_GUIDE_INDEX/#evaluate","text":"accuracy_score bias_variance_decomp bootstrap bootstrap_point632_score BootstrapOutOfBag cochrans_q combined_ftest_5x2cv confusion_matrix create_counterfactual feature_importance_permutation ftest GroupTimeSeriesSplit lift_score mcnemar_table mcnemar_tables mcnemar paired_ttest_5x2cv paired_ttest_kfold_cv paired_ttest_resampled permutation_test PredefinedHoldoutSplit proportion_difference RandomHoldoutSplit scoring","title":"evaluate"},{"location":"USER_GUIDE_INDEX/#feature_extraction","text":"LinearDiscriminantAnalysis PrincipalComponentAnalysis RBFKernelPCA","title":"feature_extraction"},{"location":"USER_GUIDE_INDEX/#feature_selection","text":"ColumnSelector ExhaustiveFeatureSelector SequentialFeatureSelector","title":"feature_selection"},{"location":"USER_GUIDE_INDEX/#file_io","text":"find_filegroups find_files","title":"file_io"},{"location":"USER_GUIDE_INDEX/#frequent_patterns","text":"apriori association_rules fpgrowth fpmax","title":"frequent_patterns"},{"location":"USER_GUIDE_INDEX/#math","text":"num_combinations num_permutations","title":"math"},{"location":"USER_GUIDE_INDEX/#plotting","text":"category_scatter checkerboard_plot ecdf enrichment_plot heatmap plot_confusion_matrix plot_pca_correlation_graph plot_decision_regions plot_learning_curves plot_linear_regression plot_sequential_feature_selection scatterplotmatrix scatter_hist stacked_barplot","title":"plotting"},{"location":"USER_GUIDE_INDEX/#preprocessing","text":"CopyTransformer DenseTransformer MeanCenterer minmax_scaling one-hot_encoding shuffle_arrays_unison standardize TransactionEncoder","title":"preprocessing"},{"location":"USER_GUIDE_INDEX/#regressor","text":"LinearRegression StackingCVRegressor StackingRegressor","title":"regressor"},{"location":"USER_GUIDE_INDEX/#text","text":"generalize_names generalize_names_duplcheck tokenizer","title":"text"},{"location":"USER_GUIDE_INDEX/#utils","text":"Counter","title":"utils"},{"location":"cite/","text":"Citing mlxtend If you use mlxtend as part of your workflow in a scientific publication, please consider citing the mlxtend repository with the following DOI: Raschka, Sebastian (2018) MLxtend: Providing machine learning and data science utilities and extensions to Python's scientific computing stack . J Open Source Softw 3(24). @article{raschkas_2018_mlxtend, author = {Sebastian Raschka}, title = {MLxtend: Providing machine learning and data science utilities and extensions to Python\u2019s scientific computing stack}, journal = {The Journal of Open Source Software}, volume = {3}, number = {24}, month = apr, year = 2018, publisher = {The Open Journal}, doi = {10.21105/joss.00638}, url = {https://joss.theoj.org/papers/10.21105/joss.00638} }","title":"Citing Mlxtend"},{"location":"cite/#citing-mlxtend","text":"If you use mlxtend as part of your workflow in a scientific publication, please consider citing the mlxtend repository with the following DOI: Raschka, Sebastian (2018) MLxtend: Providing machine learning and data science utilities and extensions to Python's scientific computing stack . J Open Source Softw 3(24). @article{raschkas_2018_mlxtend, author = {Sebastian Raschka}, title = {MLxtend: Providing machine learning and data science utilities and extensions to Python\u2019s scientific computing stack}, journal = {The Journal of Open Source Software}, volume = {3}, number = {24}, month = apr, year = 2018, publisher = {The Open Journal}, doi = {10.21105/joss.00638}, url = {https://joss.theoj.org/papers/10.21105/joss.00638} }","title":"Citing mlxtend"},{"location":"contributors/","text":"Contributors For the current list of contributors to mlxtend, please see the GitHub contributor page at https://github.com/rasbt/mlxtend/graphs/contributors .","title":"Contributors"},{"location":"contributors/#contributors","text":"For the current list of contributors to mlxtend, please see the GitHub contributor page at https://github.com/rasbt/mlxtend/graphs/contributors .","title":"Contributors"},{"location":"discuss/","text":"Discuss Any questions or comments about mlxtend? Join the mlxtend mailing list on Google Groups!","title":"Discuss"},{"location":"discuss/#discuss","text":"Any questions or comments about mlxtend? Join the mlxtend mailing list on Google Groups!","title":"Discuss"},{"location":"installation/","text":"Installing mlxtend PyPI To install mlxtend, just execute pip install mlxtend Alternatively, you download the package manually from the Python Package Index https://pypi.python.org/pypi/mlxtend , unzip it, navigate into the package, and use the following command from inside the mlxtend folder: pip install . Upgrading via pip To upgrade an existing version of mlxtend from PyPI, execute pip install mlxtend --upgrade --no-deps Please note that the dependencies (NumPy and SciPy) will also be upgraded if you omit the --no-deps flag; use the --no-deps (\"no dependencies\") flag if you don't want this. Installing mlxtend from the source distribution In rare cases, users reported problems on certain systems with the default pip installation command, which installs mlxtend from the binary distribution (\"wheels\") on PyPI. If you should encounter similar problems, you could try to install mlxtend from the source distribution instead via pip install --no-binary :all: mlxtend Also, I would appreciate it if you could report any issues that occur when using pip install mlxtend in hope that we can fix these in future releases. Conda The mlxtend package is also available through conda forge . To install mlxtend using conda, use the following command: conda install mlxtend --channel conda-forge or simply conda install mlxtend if you added conda-forge to your channels ( conda config --add channels conda-forge ). Dev Version The mlxtend version on PyPI may always one step behind; you can install the latest development version from the GitHub repository by executing pip install git+git://github.com/rasbt/mlxtend.git Or, you can fork the GitHub repository from https://github.com/rasbt/mlxtend and install mlxtend from your local drive via pip install .","title":"Installation"},{"location":"installation/#installing-mlxtend","text":"","title":"Installing mlxtend"},{"location":"installation/#pypi","text":"To install mlxtend, just execute pip install mlxtend Alternatively, you download the package manually from the Python Package Index https://pypi.python.org/pypi/mlxtend , unzip it, navigate into the package, and use the following command from inside the mlxtend folder: pip install .","title":"PyPI"},{"location":"installation/#upgrading-via-pip","text":"To upgrade an existing version of mlxtend from PyPI, execute pip install mlxtend --upgrade --no-deps Please note that the dependencies (NumPy and SciPy) will also be upgraded if you omit the --no-deps flag; use the --no-deps (\"no dependencies\") flag if you don't want this.","title":"Upgrading via pip"},{"location":"installation/#installing-mlxtend-from-the-source-distribution","text":"In rare cases, users reported problems on certain systems with the default pip installation command, which installs mlxtend from the binary distribution (\"wheels\") on PyPI. If you should encounter similar problems, you could try to install mlxtend from the source distribution instead via pip install --no-binary :all: mlxtend Also, I would appreciate it if you could report any issues that occur when using pip install mlxtend in hope that we can fix these in future releases.","title":"Installing mlxtend from the source distribution"},{"location":"installation/#conda","text":"The mlxtend package is also available through conda forge . To install mlxtend using conda, use the following command: conda install mlxtend --channel conda-forge or simply conda install mlxtend if you added conda-forge to your channels ( conda config --add channels conda-forge ).","title":"Conda"},{"location":"installation/#dev-version","text":"The mlxtend version on PyPI may always one step behind; you can install the latest development version from the GitHub repository by executing pip install git+git://github.com/rasbt/mlxtend.git Or, you can fork the GitHub repository from https://github.com/rasbt/mlxtend and install mlxtend from your local drive via pip install .","title":"Dev Version"},{"location":"license/","text":"This project is released under a permissive new BSD open source license and commercially usable. There is no warranty; not even for merchantability or fitness for a particular purpose. In addition, you may use, copy, modify, and redistribute all artistic creative works (figures and images) included in this distribution under the directory according to the terms and conditions of the Creative Commons Attribution 4.0 International License. (Computer-generated graphics such as the plots produced by matplotlib fall under the BSD license mentioned above). new BSD License New BSD License Copyright (c) 2014-2024, Sebastian Raschka. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. Neither the name of mlxtend nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \"AS IS\" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. Creative Commons Attribution 4.0 International License mlxtend documentation figures are licensed under a Creative Commons Attribution 4.0 International License. https://creativecommons.org/licenses/by-sa/4.0/ . You are free to: Share \u2014 copy and redistribute the material in any medium or format Adapt \u2014 remix, transform, and build upon the material for any purpose, even commercially. The licensor cannot revoke these freedoms as long as you follow the license terms. Under the following terms: Attribution \u2014 You must give appropriate credit, provide a link to the license, and indicate if changes were made. You may do so in any reasonable manner, but not in any way that suggests the licensor endorses you or your use. No additional restrictions \u2014 You may not apply legal terms or technological measures that legally restrict others from doing anything the license permits.","title":"License"},{"location":"license/#new-bsd-license","text":"New BSD License Copyright (c) 2014-2024, Sebastian Raschka. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. Neither the name of mlxtend nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \"AS IS\" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.","title":"new BSD License"},{"location":"license/#creative-commons-attribution-40-international-license","text":"mlxtend documentation figures are licensed under a Creative Commons Attribution 4.0 International License. https://creativecommons.org/licenses/by-sa/4.0/ .","title":"Creative Commons Attribution 4.0 International License"},{"location":"license/#you-are-free-to","text":"Share \u2014 copy and redistribute the material in any medium or format Adapt \u2014 remix, transform, and build upon the material for any purpose, even commercially. The licensor cannot revoke these freedoms as long as you follow the license terms.","title":"You are free to:"},{"location":"license/#under-the-following-terms","text":"Attribution \u2014 You must give appropriate credit, provide a link to the license, and indicate if changes were made. You may do so in any reasonable manner, but not in any way that suggests the licensor endorses you or your use. No additional restrictions \u2014 You may not apply legal terms or technological measures that legally restrict others from doing anything the license permits.","title":"Under the following terms:"},{"location":"api_modules/mlxtend.classifier/Adaline/","text":"Adaline Adaline(eta=0.01, epochs=50, minibatches=None, random_seed=None, print_progress=0) ADAptive LInear NEuron classifier. Note that this implementation of Adaline expects binary class labels in {0, 1}. Parameters eta : float (default: 0.01) solver rate (between 0.0 and 1.0) epochs : int (default: 50) Passes over the training dataset. Prior to each epoch, the dataset is shuffled if minibatches > 1 to prevent cycles in stochastic gradient descent. minibatches : int (default: None) The number of minibatches for gradient-based optimization. If None: Normal Equations (closed-form solution) If 1: Gradient Descent learning If len(y): Stochastic Gradient Descent (SGD) online learning If 1 < minibatches < len(y): SGD Minibatch learning random_seed : int (default: None) Set random state for shuffling and initializing the weights. print_progress : int (default: 0) Prints progress in fitting to stderr if not solver='normal equation' 0: No output 1: Epochs elapsed and cost 2: 1 plus time elapsed 3: 2 plus estimated time until completion Attributes w_ : 2d-array, shape={n_features, 1} Model weights after fitting. b_ : 1d-array, shape={1,} Bias unit after fitting. cost_ : list Sum of squared errors after each epoch. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/classifier/Adaline/ Methods fit(X, y, init_params=True) Learn model from training data. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values. init_params : bool (default: True) Re-initializes model parameters prior to fitting. Set False to continue training with weights from a previous model fitting. Returns self : object get_params(deep=True) Get parameters for this estimator. Parameters deep : boolean, optional If True, will return the parameters for this estimator and contained subobjects that are estimators. Returns params : mapping of string to any Parameter names mapped to their values.' adapted from https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/base.py Author: Gael Varoquaux gael.varoquaux@normalesup.org License: BSD 3 clause predict(X) Predict targets from X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns target_values : array-like, shape = [n_samples] Predicted target values. score(X, y) Compute the prediction accuracy Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values (true class labels). Returns acc : float The prediction accuracy as a float between 0.0 and 1.0 (perfect score). set_params( params) Set the parameters of this estimator. The method works on simple estimators as well as on nested objects (such as pipelines). The latter have parameters of the form <component>__<parameter> so that it's possible to update each component of a nested object. Returns self adapted from https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/base.py Author: Gael Varoquaux <gael.varoquaux@normalesup.org> License: BSD 3 clause","title":"Adaline"},{"location":"api_modules/mlxtend.classifier/Adaline/#adaline","text":"Adaline(eta=0.01, epochs=50, minibatches=None, random_seed=None, print_progress=0) ADAptive LInear NEuron classifier. Note that this implementation of Adaline expects binary class labels in {0, 1}. Parameters eta : float (default: 0.01) solver rate (between 0.0 and 1.0) epochs : int (default: 50) Passes over the training dataset. Prior to each epoch, the dataset is shuffled if minibatches > 1 to prevent cycles in stochastic gradient descent. minibatches : int (default: None) The number of minibatches for gradient-based optimization. If None: Normal Equations (closed-form solution) If 1: Gradient Descent learning If len(y): Stochastic Gradient Descent (SGD) online learning If 1 < minibatches < len(y): SGD Minibatch learning random_seed : int (default: None) Set random state for shuffling and initializing the weights. print_progress : int (default: 0) Prints progress in fitting to stderr if not solver='normal equation' 0: No output 1: Epochs elapsed and cost 2: 1 plus time elapsed 3: 2 plus estimated time until completion Attributes w_ : 2d-array, shape={n_features, 1} Model weights after fitting. b_ : 1d-array, shape={1,} Bias unit after fitting. cost_ : list Sum of squared errors after each epoch. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/classifier/Adaline/","title":"Adaline"},{"location":"api_modules/mlxtend.classifier/Adaline/#methods","text":"fit(X, y, init_params=True) Learn model from training data. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values. init_params : bool (default: True) Re-initializes model parameters prior to fitting. Set False to continue training with weights from a previous model fitting. Returns self : object get_params(deep=True) Get parameters for this estimator. Parameters deep : boolean, optional If True, will return the parameters for this estimator and contained subobjects that are estimators. Returns params : mapping of string to any Parameter names mapped to their values.' adapted from https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/base.py Author: Gael Varoquaux gael.varoquaux@normalesup.org License: BSD 3 clause predict(X) Predict targets from X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns target_values : array-like, shape = [n_samples] Predicted target values. score(X, y) Compute the prediction accuracy Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values (true class labels). Returns acc : float The prediction accuracy as a float between 0.0 and 1.0 (perfect score). set_params( params) Set the parameters of this estimator. The method works on simple estimators as well as on nested objects (such as pipelines). The latter have parameters of the form <component>__<parameter> so that it's possible to update each component of a nested object. Returns self adapted from https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/base.py Author: Gael Varoquaux <gael.varoquaux@normalesup.org> License: BSD 3 clause","title":"Methods"},{"location":"api_modules/mlxtend.classifier/EnsembleVoteClassifier/","text":"EnsembleVoteClassifier EnsembleVoteClassifier(clfs, voting='hard', weights=None, verbose=0, use_clones=True, fit_base_estimators=True) Soft Voting/Majority Rule classifier for scikit-learn estimators. Parameters clfs : array-like, shape = [n_classifiers] A list of classifiers. Invoking the fit method on the VotingClassifier will fit clones of those original classifiers be stored in the class attribute if use_clones=True (default) and fit_base_estimators=True (default). voting : str, {'hard', 'soft'} (default='hard') If 'hard', uses predicted class labels for majority rule voting. Else if 'soft', predicts the class label based on the argmax of the sums of the predicted probalities, which is recommended for an ensemble of well-calibrated classifiers. weights : array-like, shape = [n_classifiers], optional (default= None ) Sequence of weights ( float or int ) to weight the occurances of predicted class labels ( hard voting) or class probabilities before averaging ( soft voting). Uses uniform weights if None . verbose : int, optional (default=0) Controls the verbosity of the building process. - verbose=0 (default): Prints nothing - verbose=1 : Prints the number & name of the clf being fitted - verbose=2 : Prints info about the parameters of the clf being fitted - verbose>2 : Changes verbose param of the underlying clf to self.verbose - 2 use_clones : bool (default: True) Clones the classifiers for stacking classification if True (default) or else uses the original ones, which will be refitted on the dataset upon calling the fit method. Hence, if use_clones=True, the original input classifiers will remain unmodified upon using the StackingClassifier's fit method. Setting use_clones=False is recommended if you are working with estimators that are supporting the scikit-learn fit/predict API interface but are not compatible to scikit-learn's clone function. fit_base_estimators : bool (default: True) Refits classifiers in clfs if True; uses references to the clfs , otherwise (assumes that the classifiers were already fit). Note: fit_base_estimators=False will enforce use_clones to be False, and is incompatible to most scikit-learn wrappers! For instance, if any form of cross-validation is performed this would require the re-fitting classifiers to training folds, which would raise a NotFitterError if fit_base_estimators=False. (New in mlxtend v0.6.) Attributes classes_ : array-like, shape = [n_predictions] clf : array-like, shape = [n_predictions] The input classifiers; may be overwritten if use_clones=False clf_ : array-like, shape = [n_predictions] Fitted input classifiers; clones if use_clones=True Examples ``` >>> import numpy as np >>> from sklearn.linear_model import LogisticRegression >>> from sklearn.naive_bayes import GaussianNB >>> from sklearn.ensemble import RandomForestClassifier >>> from mlxtend.sklearn import EnsembleVoteClassifier >>> clf1 = LogisticRegression(random_seed=1) >>> clf2 = RandomForestClassifier(random_seed=1) >>> clf3 = GaussianNB() >>> X = np.array([[-1, -1], [-2, -1], [-3, -2], [1, 1], [2, 1], [3, 2]]) >>> y = np.array([1, 1, 1, 2, 2, 2]) >>> eclf1 = EnsembleVoteClassifier(clfs=[clf1, clf2, clf3], ... voting='hard', verbose=1) >>> eclf1 = eclf1.fit(X, y) >>> print(eclf1.predict(X)) [1 1 1 2 2 2] >>> eclf2 = EnsembleVoteClassifier(clfs=[clf1, clf2, clf3], voting='soft') >>> eclf2 = eclf2.fit(X, y) >>> print(eclf2.predict(X)) [1 1 1 2 2 2] >>> eclf3 = EnsembleVoteClassifier(clfs=[clf1, clf2, clf3], ... voting='soft', weights=[2,1,1]) >>> eclf3 = eclf3.fit(X, y) >>> print(eclf3.predict(X)) [1 1 1 2 2 2] >>> For more usage examples, please see https://rasbt.github.io/mlxtend/user_guide/classifier/EnsembleVoteClassifier/ ``` Methods fit(X, y, sample_weight=None) Learn weight coefficients from training data for each classifier. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values. sample_weight : array-like, shape = [n_samples], optional Sample weights passed as sample_weights to each regressor in the regressors list as well as the meta_regressor. Raises error if some regressor does not support sample_weight in the fit() method. Returns self : object fit_transform(X, y=None, fit_params) Fit to data, then transform it. Fits transformer to `X` and `y` with optional parameters `fit_params` and returns a transformed version of `X`. Parameters X : array-like of shape (n_samples, n_features) Input samples. y : array-like of shape (n_samples,) or (n_samples, n_outputs), default=None Target values (None for unsupervised transformations). **fit_params : dict Additional fit parameters. Returns X_new : ndarray array of shape (n_samples, n_features_new) Transformed array. get_params(deep=True) Return estimator parameter names for GridSearch support. predict(X) Predict class labels for X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns maj : array-like, shape = [n_samples] Predicted class labels. predict_proba(X) Predict class probabilities for X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns avg : array-like, shape = [n_samples, n_classes] Weighted average probability for each class per sample. score(X, y, sample_weight=None) Return the mean accuracy on the given test data and labels. In multi-label classification, this is the subset accuracy which is a harsh metric since you require for each sample that each label set be correctly predicted. Parameters X : array-like of shape (n_samples, n_features) Test samples. y : array-like of shape (n_samples,) or (n_samples, n_outputs) True labels for X . sample_weight : array-like of shape (n_samples,), default=None Sample weights. Returns score : float Mean accuracy of self.predict(X) w.r.t. y . set_output( , transform=None)* Set output container. See :ref:`sphx_glr_auto_examples_miscellaneous_plot_set_output.py` for an example on how to use the API. Parameters transform : {\"default\", \"pandas\"}, default=None Configure output of transform and fit_transform . \"default\" : Default output format of a transformer \"pandas\" : DataFrame output None : Transform configuration is unchanged Returns self : estimator instance Estimator instance. set_params( params) Set the parameters of this estimator. The method works on simple estimators as well as on nested objects (such as :class:`~sklearn.pipeline.Pipeline`). The latter have parameters of the form ``<component>__<parameter>`` so that it's possible to update each component of a nested object. Parameters **params : dict Estimator parameters. Returns self : estimator instance Estimator instance. transform(X) Return class labels or probabilities for X for each estimator. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns If voting='soft'`` : array-like = [n_classifiers, n_samples, n_classes] Class probabilties calculated by each classifier. If voting='hard'`` : array-like = [n_classifiers, n_samples] Class labels predicted by each classifier.","title":"EnsembleVoteClassifier"},{"location":"api_modules/mlxtend.classifier/EnsembleVoteClassifier/#ensemblevoteclassifier","text":"EnsembleVoteClassifier(clfs, voting='hard', weights=None, verbose=0, use_clones=True, fit_base_estimators=True) Soft Voting/Majority Rule classifier for scikit-learn estimators. Parameters clfs : array-like, shape = [n_classifiers] A list of classifiers. Invoking the fit method on the VotingClassifier will fit clones of those original classifiers be stored in the class attribute if use_clones=True (default) and fit_base_estimators=True (default). voting : str, {'hard', 'soft'} (default='hard') If 'hard', uses predicted class labels for majority rule voting. Else if 'soft', predicts the class label based on the argmax of the sums of the predicted probalities, which is recommended for an ensemble of well-calibrated classifiers. weights : array-like, shape = [n_classifiers], optional (default= None ) Sequence of weights ( float or int ) to weight the occurances of predicted class labels ( hard voting) or class probabilities before averaging ( soft voting). Uses uniform weights if None . verbose : int, optional (default=0) Controls the verbosity of the building process. - verbose=0 (default): Prints nothing - verbose=1 : Prints the number & name of the clf being fitted - verbose=2 : Prints info about the parameters of the clf being fitted - verbose>2 : Changes verbose param of the underlying clf to self.verbose - 2 use_clones : bool (default: True) Clones the classifiers for stacking classification if True (default) or else uses the original ones, which will be refitted on the dataset upon calling the fit method. Hence, if use_clones=True, the original input classifiers will remain unmodified upon using the StackingClassifier's fit method. Setting use_clones=False is recommended if you are working with estimators that are supporting the scikit-learn fit/predict API interface but are not compatible to scikit-learn's clone function. fit_base_estimators : bool (default: True) Refits classifiers in clfs if True; uses references to the clfs , otherwise (assumes that the classifiers were already fit). Note: fit_base_estimators=False will enforce use_clones to be False, and is incompatible to most scikit-learn wrappers! For instance, if any form of cross-validation is performed this would require the re-fitting classifiers to training folds, which would raise a NotFitterError if fit_base_estimators=False. (New in mlxtend v0.6.) Attributes classes_ : array-like, shape = [n_predictions] clf : array-like, shape = [n_predictions] The input classifiers; may be overwritten if use_clones=False clf_ : array-like, shape = [n_predictions] Fitted input classifiers; clones if use_clones=True Examples ``` >>> import numpy as np >>> from sklearn.linear_model import LogisticRegression >>> from sklearn.naive_bayes import GaussianNB >>> from sklearn.ensemble import RandomForestClassifier >>> from mlxtend.sklearn import EnsembleVoteClassifier >>> clf1 = LogisticRegression(random_seed=1) >>> clf2 = RandomForestClassifier(random_seed=1) >>> clf3 = GaussianNB() >>> X = np.array([[-1, -1], [-2, -1], [-3, -2], [1, 1], [2, 1], [3, 2]]) >>> y = np.array([1, 1, 1, 2, 2, 2]) >>> eclf1 = EnsembleVoteClassifier(clfs=[clf1, clf2, clf3], ... voting='hard', verbose=1) >>> eclf1 = eclf1.fit(X, y) >>> print(eclf1.predict(X)) [1 1 1 2 2 2] >>> eclf2 = EnsembleVoteClassifier(clfs=[clf1, clf2, clf3], voting='soft') >>> eclf2 = eclf2.fit(X, y) >>> print(eclf2.predict(X)) [1 1 1 2 2 2] >>> eclf3 = EnsembleVoteClassifier(clfs=[clf1, clf2, clf3], ... voting='soft', weights=[2,1,1]) >>> eclf3 = eclf3.fit(X, y) >>> print(eclf3.predict(X)) [1 1 1 2 2 2] >>> For more usage examples, please see https://rasbt.github.io/mlxtend/user_guide/classifier/EnsembleVoteClassifier/ ```","title":"EnsembleVoteClassifier"},{"location":"api_modules/mlxtend.classifier/EnsembleVoteClassifier/#methods","text":"fit(X, y, sample_weight=None) Learn weight coefficients from training data for each classifier. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values. sample_weight : array-like, shape = [n_samples], optional Sample weights passed as sample_weights to each regressor in the regressors list as well as the meta_regressor. Raises error if some regressor does not support sample_weight in the fit() method. Returns self : object fit_transform(X, y=None, fit_params) Fit to data, then transform it. Fits transformer to `X` and `y` with optional parameters `fit_params` and returns a transformed version of `X`. Parameters X : array-like of shape (n_samples, n_features) Input samples. y : array-like of shape (n_samples,) or (n_samples, n_outputs), default=None Target values (None for unsupervised transformations). **fit_params : dict Additional fit parameters. Returns X_new : ndarray array of shape (n_samples, n_features_new) Transformed array. get_params(deep=True) Return estimator parameter names for GridSearch support. predict(X) Predict class labels for X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns maj : array-like, shape = [n_samples] Predicted class labels. predict_proba(X) Predict class probabilities for X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns avg : array-like, shape = [n_samples, n_classes] Weighted average probability for each class per sample. score(X, y, sample_weight=None) Return the mean accuracy on the given test data and labels. In multi-label classification, this is the subset accuracy which is a harsh metric since you require for each sample that each label set be correctly predicted. Parameters X : array-like of shape (n_samples, n_features) Test samples. y : array-like of shape (n_samples,) or (n_samples, n_outputs) True labels for X . sample_weight : array-like of shape (n_samples,), default=None Sample weights. Returns score : float Mean accuracy of self.predict(X) w.r.t. y . set_output( , transform=None)* Set output container. See :ref:`sphx_glr_auto_examples_miscellaneous_plot_set_output.py` for an example on how to use the API. Parameters transform : {\"default\", \"pandas\"}, default=None Configure output of transform and fit_transform . \"default\" : Default output format of a transformer \"pandas\" : DataFrame output None : Transform configuration is unchanged Returns self : estimator instance Estimator instance. set_params( params) Set the parameters of this estimator. The method works on simple estimators as well as on nested objects (such as :class:`~sklearn.pipeline.Pipeline`). The latter have parameters of the form ``<component>__<parameter>`` so that it's possible to update each component of a nested object. Parameters **params : dict Estimator parameters. Returns self : estimator instance Estimator instance. transform(X) Return class labels or probabilities for X for each estimator. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns If voting='soft'`` : array-like = [n_classifiers, n_samples, n_classes] Class probabilties calculated by each classifier. If voting='hard'`` : array-like = [n_classifiers, n_samples] Class labels predicted by each classifier.","title":"Methods"},{"location":"api_modules/mlxtend.classifier/LogisticRegression/","text":"LogisticRegression LogisticRegression(eta=0.01, epochs=50, l2_lambda=0.0, minibatches=1, random_seed=None, print_progress=0) Logistic regression classifier. Note that this implementation of Logistic Regression expects binary class labels in {0, 1}. Parameters eta : float (default: 0.01) Learning rate (between 0.0 and 1.0) epochs : int (default: 50) Passes over the training dataset. Prior to each epoch, the dataset is shuffled if minibatches > 1 to prevent cycles in stochastic gradient descent. l2_lambda : float Regularization parameter for L2 regularization. No regularization if l2_lambda=0.0. minibatches : int (default: 1) The number of minibatches for gradient-based optimization. If 1: Gradient Descent learning If len(y): Stochastic Gradient Descent (SGD) online learning If 1 < minibatches < len(y): SGD Minibatch learning random_seed : int (default: None) Set random state for shuffling and initializing the weights. print_progress : int (default: 0) Prints progress in fitting to stderr. 0: No output 1: Epochs elapsed and cost 2: 1 plus time elapsed 3: 2 plus estimated time until completion Attributes w_ : 2d-array, shape={n_features, 1} Model weights after fitting. b_ : 1d-array, shape={1,} Bias unit after fitting. cost_ : list List of floats with cross_entropy cost (sgd or gd) for every epoch. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/classifier/LogisticRegression/ Methods fit(X, y, init_params=True) Learn model from training data. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values. init_params : bool (default: True) Re-initializes model parameters prior to fitting. Set False to continue training with weights from a previous model fitting. Returns self : object get_params(deep=True) Get parameters for this estimator. Parameters deep : boolean, optional If True, will return the parameters for this estimator and contained subobjects that are estimators. Returns params : mapping of string to any Parameter names mapped to their values.' adapted from https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/base.py Author: Gael Varoquaux gael.varoquaux@normalesup.org License: BSD 3 clause predict(X) Predict targets from X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns target_values : array-like, shape = [n_samples] Predicted target values. predict_proba(X) Predict class probabilities of X from the net input. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns Class 1 probability : float score(X, y) Compute the prediction accuracy Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values (true class labels). Returns acc : float The prediction accuracy as a float between 0.0 and 1.0 (perfect score). set_params( params) Set the parameters of this estimator. The method works on simple estimators as well as on nested objects (such as pipelines). The latter have parameters of the form <component>__<parameter> so that it's possible to update each component of a nested object. Returns self adapted from https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/base.py Author: Gael Varoquaux <gael.varoquaux@normalesup.org> License: BSD 3 clause","title":"LogisticRegression"},{"location":"api_modules/mlxtend.classifier/LogisticRegression/#logisticregression","text":"LogisticRegression(eta=0.01, epochs=50, l2_lambda=0.0, minibatches=1, random_seed=None, print_progress=0) Logistic regression classifier. Note that this implementation of Logistic Regression expects binary class labels in {0, 1}. Parameters eta : float (default: 0.01) Learning rate (between 0.0 and 1.0) epochs : int (default: 50) Passes over the training dataset. Prior to each epoch, the dataset is shuffled if minibatches > 1 to prevent cycles in stochastic gradient descent. l2_lambda : float Regularization parameter for L2 regularization. No regularization if l2_lambda=0.0. minibatches : int (default: 1) The number of minibatches for gradient-based optimization. If 1: Gradient Descent learning If len(y): Stochastic Gradient Descent (SGD) online learning If 1 < minibatches < len(y): SGD Minibatch learning random_seed : int (default: None) Set random state for shuffling and initializing the weights. print_progress : int (default: 0) Prints progress in fitting to stderr. 0: No output 1: Epochs elapsed and cost 2: 1 plus time elapsed 3: 2 plus estimated time until completion Attributes w_ : 2d-array, shape={n_features, 1} Model weights after fitting. b_ : 1d-array, shape={1,} Bias unit after fitting. cost_ : list List of floats with cross_entropy cost (sgd or gd) for every epoch. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/classifier/LogisticRegression/","title":"LogisticRegression"},{"location":"api_modules/mlxtend.classifier/LogisticRegression/#methods","text":"fit(X, y, init_params=True) Learn model from training data. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values. init_params : bool (default: True) Re-initializes model parameters prior to fitting. Set False to continue training with weights from a previous model fitting. Returns self : object get_params(deep=True) Get parameters for this estimator. Parameters deep : boolean, optional If True, will return the parameters for this estimator and contained subobjects that are estimators. Returns params : mapping of string to any Parameter names mapped to their values.' adapted from https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/base.py Author: Gael Varoquaux gael.varoquaux@normalesup.org License: BSD 3 clause predict(X) Predict targets from X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns target_values : array-like, shape = [n_samples] Predicted target values. predict_proba(X) Predict class probabilities of X from the net input. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns Class 1 probability : float score(X, y) Compute the prediction accuracy Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values (true class labels). Returns acc : float The prediction accuracy as a float between 0.0 and 1.0 (perfect score). set_params( params) Set the parameters of this estimator. The method works on simple estimators as well as on nested objects (such as pipelines). The latter have parameters of the form <component>__<parameter> so that it's possible to update each component of a nested object. Returns self adapted from https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/base.py Author: Gael Varoquaux <gael.varoquaux@normalesup.org> License: BSD 3 clause","title":"Methods"},{"location":"api_modules/mlxtend.classifier/MultiLayerPerceptron/","text":"MultiLayerPerceptron MultiLayerPerceptron(eta=0.5, epochs=50, hidden_layers=[50], n_classes=None, momentum=0.0, l1=0.0, l2=0.0, dropout=1.0, decrease_const=0.0, minibatches=1, random_seed=None, print_progress=0) Multi-layer perceptron classifier with logistic sigmoid activations Parameters eta : float (default: 0.5) Learning rate (between 0.0 and 1.0) epochs : int (default: 50) Passes over the training dataset. Prior to each epoch, the dataset is shuffled if minibatches > 1 to prevent cycles in stochastic gradient descent. hidden_layers : list (default: [50]) Number of units per hidden layer. By default 50 units in the first hidden layer. At the moment only 1 hidden layer is supported n_classes : int (default: None) A positive integer to declare the number of class labels if not all class labels are present in a partial training set. Gets the number of class labels automatically if None. l1 : float (default: 0.0) L1 regularization strength l2 : float (default: 0.0) L2 regularization strength momentum : float (default: 0.0) Momentum constant. Factor multiplied with the gradient of the previous epoch t-1 to improve learning speed w(t) := w(t) - (grad(t) + momentum * grad(t-1)) decrease_const : float (default: 0.0) Decrease constant. Shrinks the learning rate after each epoch via eta / (1 + epoch*decrease_const) minibatches : int (default: 1) Divide the training data into k minibatches for accelerated stochastic gradient descent learning. Gradient Descent Learning if minibatches = 1 Stochastic Gradient Descent learning if minibatches = len(y) Minibatch learning if minibatches > 1 random_seed : int (default: None) Set random state for shuffling and initializing the weights. print_progress : int (default: 0) Prints progress in fitting to stderr. 0: No output 1: Epochs elapsed and cost 2: 1 plus time elapsed 3: 2 plus estimated time until completion Attributes w_ : 2d-array, shape=[n_features, n_classes] Weights after fitting. b_ : 1D-array, shape=[n_classes] Bias units after fitting. cost_ : list List of floats; the mean categorical cross entropy cost after each epoch. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/classifier/MultiLayerPerceptron/ Methods fit(X, y, init_params=True) Learn model from training data. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values. init_params : bool (default: True) Re-initializes model parameters prior to fitting. Set False to continue training with weights from a previous model fitting. Returns self : object get_params(deep=True) Get parameters for this estimator. Parameters deep : boolean, optional If True, will return the parameters for this estimator and contained subobjects that are estimators. Returns params : mapping of string to any Parameter names mapped to their values.' adapted from https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/base.py Author: Gael Varoquaux gael.varoquaux@normalesup.org License: BSD 3 clause predict(X) Predict targets from X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns target_values : array-like, shape = [n_samples] Predicted target values. predict_proba(X) Predict class probabilities of X from the net input. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns Class probabilties : array-like, shape= [n_samples, n_classes] score(X, y) Compute the prediction accuracy Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values (true class labels). Returns acc : float The prediction accuracy as a float between 0.0 and 1.0 (perfect score). set_params( params) Set the parameters of this estimator. The method works on simple estimators as well as on nested objects (such as pipelines). The latter have parameters of the form <component>__<parameter> so that it's possible to update each component of a nested object. Returns self adapted from https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/base.py Author: Gael Varoquaux <gael.varoquaux@normalesup.org> License: BSD 3 clause","title":"MultiLayerPerceptron"},{"location":"api_modules/mlxtend.classifier/MultiLayerPerceptron/#multilayerperceptron","text":"MultiLayerPerceptron(eta=0.5, epochs=50, hidden_layers=[50], n_classes=None, momentum=0.0, l1=0.0, l2=0.0, dropout=1.0, decrease_const=0.0, minibatches=1, random_seed=None, print_progress=0) Multi-layer perceptron classifier with logistic sigmoid activations Parameters eta : float (default: 0.5) Learning rate (between 0.0 and 1.0) epochs : int (default: 50) Passes over the training dataset. Prior to each epoch, the dataset is shuffled if minibatches > 1 to prevent cycles in stochastic gradient descent. hidden_layers : list (default: [50]) Number of units per hidden layer. By default 50 units in the first hidden layer. At the moment only 1 hidden layer is supported n_classes : int (default: None) A positive integer to declare the number of class labels if not all class labels are present in a partial training set. Gets the number of class labels automatically if None. l1 : float (default: 0.0) L1 regularization strength l2 : float (default: 0.0) L2 regularization strength momentum : float (default: 0.0) Momentum constant. Factor multiplied with the gradient of the previous epoch t-1 to improve learning speed w(t) := w(t) - (grad(t) + momentum * grad(t-1)) decrease_const : float (default: 0.0) Decrease constant. Shrinks the learning rate after each epoch via eta / (1 + epoch*decrease_const) minibatches : int (default: 1) Divide the training data into k minibatches for accelerated stochastic gradient descent learning. Gradient Descent Learning if minibatches = 1 Stochastic Gradient Descent learning if minibatches = len(y) Minibatch learning if minibatches > 1 random_seed : int (default: None) Set random state for shuffling and initializing the weights. print_progress : int (default: 0) Prints progress in fitting to stderr. 0: No output 1: Epochs elapsed and cost 2: 1 plus time elapsed 3: 2 plus estimated time until completion Attributes w_ : 2d-array, shape=[n_features, n_classes] Weights after fitting. b_ : 1D-array, shape=[n_classes] Bias units after fitting. cost_ : list List of floats; the mean categorical cross entropy cost after each epoch. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/classifier/MultiLayerPerceptron/","title":"MultiLayerPerceptron"},{"location":"api_modules/mlxtend.classifier/MultiLayerPerceptron/#methods","text":"fit(X, y, init_params=True) Learn model from training data. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values. init_params : bool (default: True) Re-initializes model parameters prior to fitting. Set False to continue training with weights from a previous model fitting. Returns self : object get_params(deep=True) Get parameters for this estimator. Parameters deep : boolean, optional If True, will return the parameters for this estimator and contained subobjects that are estimators. Returns params : mapping of string to any Parameter names mapped to their values.' adapted from https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/base.py Author: Gael Varoquaux gael.varoquaux@normalesup.org License: BSD 3 clause predict(X) Predict targets from X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns target_values : array-like, shape = [n_samples] Predicted target values. predict_proba(X) Predict class probabilities of X from the net input. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns Class probabilties : array-like, shape= [n_samples, n_classes] score(X, y) Compute the prediction accuracy Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values (true class labels). Returns acc : float The prediction accuracy as a float between 0.0 and 1.0 (perfect score). set_params( params) Set the parameters of this estimator. The method works on simple estimators as well as on nested objects (such as pipelines). The latter have parameters of the form <component>__<parameter> so that it's possible to update each component of a nested object. Returns self adapted from https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/base.py Author: Gael Varoquaux <gael.varoquaux@normalesup.org> License: BSD 3 clause","title":"Methods"},{"location":"api_modules/mlxtend.classifier/OneRClassifier/","text":"OneRClassifier OneRClassifier(resolve_ties='first') OneR (One Rule) Classifier. Parameters resolve_ties : str (default: 'first') Option for how to resolve ties if two or more features have the same error. Options are - 'first' (default): chooses first feature in the list, i.e., feature with the lower column index. - 'chi-squared': performs a chi-squared test for each feature against the target and selects the feature with the lowest p-value. Attributes self.classes_labels_ : array-like, shape = [n_labels] Array containing the unique class labels found in the training set. self.feature_idx_ : int The index of the rules' feature based on the column in the training set. self.p_value_ : float The p value for a given feature. Only available after calling fit when the OneR attribute resolve_ties = 'chi-squared' is set. self.prediction_dict_ : dict Dictionary containing information about the feature's (self.feature_idx_) rules and total error. E.g., {'total error': 37, 'rules (value: class)': {0: 0, 1: 2}} means the total error is 37, and the rules are \"if feature value == 0 classify as 0\" and \"if feature value == 1 classify as 2\". (And classify as class 1 otherwise.) For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/classifier/OneRClassifier/ Methods fit(X, y) Learn rule from training data. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values. Returns self : object get_params(deep=True) Get parameters for this estimator. Parameters deep : bool, default=True If True, will return the parameters for this estimator and contained subobjects that are estimators. Returns params : dict Parameter names mapped to their values. predict(X) Predict class labels for X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns maj : array-like, shape = [n_samples] Predicted class labels. score(X, y, sample_weight=None) Return the mean accuracy on the given test data and labels. In multi-label classification, this is the subset accuracy which is a harsh metric since you require for each sample that each label set be correctly predicted. Parameters X : array-like of shape (n_samples, n_features) Test samples. y : array-like of shape (n_samples,) or (n_samples, n_outputs) True labels for X . sample_weight : array-like of shape (n_samples,), default=None Sample weights. Returns score : float Mean accuracy of self.predict(X) w.r.t. y . set_params( params) Set the parameters of this estimator. The method works on simple estimators as well as on nested objects (such as :class:`~sklearn.pipeline.Pipeline`). The latter have parameters of the form ``<component>__<parameter>`` so that it's possible to update each component of a nested object. Parameters **params : dict Estimator parameters. Returns self : estimator instance Estimator instance.","title":"OneRClassifier"},{"location":"api_modules/mlxtend.classifier/OneRClassifier/#onerclassifier","text":"OneRClassifier(resolve_ties='first') OneR (One Rule) Classifier. Parameters resolve_ties : str (default: 'first') Option for how to resolve ties if two or more features have the same error. Options are - 'first' (default): chooses first feature in the list, i.e., feature with the lower column index. - 'chi-squared': performs a chi-squared test for each feature against the target and selects the feature with the lowest p-value. Attributes self.classes_labels_ : array-like, shape = [n_labels] Array containing the unique class labels found in the training set. self.feature_idx_ : int The index of the rules' feature based on the column in the training set. self.p_value_ : float The p value for a given feature. Only available after calling fit when the OneR attribute resolve_ties = 'chi-squared' is set. self.prediction_dict_ : dict Dictionary containing information about the feature's (self.feature_idx_) rules and total error. E.g., {'total error': 37, 'rules (value: class)': {0: 0, 1: 2}} means the total error is 37, and the rules are \"if feature value == 0 classify as 0\" and \"if feature value == 1 classify as 2\". (And classify as class 1 otherwise.) For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/classifier/OneRClassifier/","title":"OneRClassifier"},{"location":"api_modules/mlxtend.classifier/OneRClassifier/#methods","text":"fit(X, y) Learn rule from training data. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values. Returns self : object get_params(deep=True) Get parameters for this estimator. Parameters deep : bool, default=True If True, will return the parameters for this estimator and contained subobjects that are estimators. Returns params : dict Parameter names mapped to their values. predict(X) Predict class labels for X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns maj : array-like, shape = [n_samples] Predicted class labels. score(X, y, sample_weight=None) Return the mean accuracy on the given test data and labels. In multi-label classification, this is the subset accuracy which is a harsh metric since you require for each sample that each label set be correctly predicted. Parameters X : array-like of shape (n_samples, n_features) Test samples. y : array-like of shape (n_samples,) or (n_samples, n_outputs) True labels for X . sample_weight : array-like of shape (n_samples,), default=None Sample weights. Returns score : float Mean accuracy of self.predict(X) w.r.t. y . set_params( params) Set the parameters of this estimator. The method works on simple estimators as well as on nested objects (such as :class:`~sklearn.pipeline.Pipeline`). The latter have parameters of the form ``<component>__<parameter>`` so that it's possible to update each component of a nested object. Parameters **params : dict Estimator parameters. Returns self : estimator instance Estimator instance.","title":"Methods"},{"location":"api_modules/mlxtend.classifier/Perceptron/","text":"Perceptron Perceptron(eta=0.1, epochs=50, random_seed=None, print_progress=0) Perceptron classifier. Note that this implementation of the Perceptron expects binary class labels in {0, 1}. Parameters eta : float (default: 0.1) Learning rate (between 0.0 and 1.0) epochs : int (default: 50) Number of passes over the training dataset. Prior to each epoch, the dataset is shuffled to prevent cycles. random_seed : int Random state for initializing random weights and shuffling. print_progress : int (default: 0) Prints progress in fitting to stderr. 0: No output 1: Epochs elapsed and cost 2: 1 plus time elapsed 3: 2 plus estimated time until completion Attributes w_ : 2d-array, shape={n_features, 1} Model weights after fitting. b_ : 1d-array, shape={1,} Bias unit after fitting. cost_ : list Number of misclassifications in every epoch. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/classifier/Perceptron/ Methods fit(X, y, init_params=True) Learn model from training data. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values. init_params : bool (default: True) Re-initializes model parameters prior to fitting. Set False to continue training with weights from a previous model fitting. Returns self : object get_params(deep=True) Get parameters for this estimator. Parameters deep : boolean, optional If True, will return the parameters for this estimator and contained subobjects that are estimators. Returns params : mapping of string to any Parameter names mapped to their values.' adapted from https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/base.py Author: Gael Varoquaux gael.varoquaux@normalesup.org License: BSD 3 clause predict(X) Predict targets from X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns target_values : array-like, shape = [n_samples] Predicted target values. score(X, y) Compute the prediction accuracy Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values (true class labels). Returns acc : float The prediction accuracy as a float between 0.0 and 1.0 (perfect score). set_params( params) Set the parameters of this estimator. The method works on simple estimators as well as on nested objects (such as pipelines). The latter have parameters of the form <component>__<parameter> so that it's possible to update each component of a nested object. Returns self adapted from https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/base.py Author: Gael Varoquaux <gael.varoquaux@normalesup.org> License: BSD 3 clause","title":"Perceptron"},{"location":"api_modules/mlxtend.classifier/Perceptron/#perceptron","text":"Perceptron(eta=0.1, epochs=50, random_seed=None, print_progress=0) Perceptron classifier. Note that this implementation of the Perceptron expects binary class labels in {0, 1}. Parameters eta : float (default: 0.1) Learning rate (between 0.0 and 1.0) epochs : int (default: 50) Number of passes over the training dataset. Prior to each epoch, the dataset is shuffled to prevent cycles. random_seed : int Random state for initializing random weights and shuffling. print_progress : int (default: 0) Prints progress in fitting to stderr. 0: No output 1: Epochs elapsed and cost 2: 1 plus time elapsed 3: 2 plus estimated time until completion Attributes w_ : 2d-array, shape={n_features, 1} Model weights after fitting. b_ : 1d-array, shape={1,} Bias unit after fitting. cost_ : list Number of misclassifications in every epoch. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/classifier/Perceptron/","title":"Perceptron"},{"location":"api_modules/mlxtend.classifier/Perceptron/#methods","text":"fit(X, y, init_params=True) Learn model from training data. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values. init_params : bool (default: True) Re-initializes model parameters prior to fitting. Set False to continue training with weights from a previous model fitting. Returns self : object get_params(deep=True) Get parameters for this estimator. Parameters deep : boolean, optional If True, will return the parameters for this estimator and contained subobjects that are estimators. Returns params : mapping of string to any Parameter names mapped to their values.' adapted from https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/base.py Author: Gael Varoquaux gael.varoquaux@normalesup.org License: BSD 3 clause predict(X) Predict targets from X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns target_values : array-like, shape = [n_samples] Predicted target values. score(X, y) Compute the prediction accuracy Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values (true class labels). Returns acc : float The prediction accuracy as a float between 0.0 and 1.0 (perfect score). set_params( params) Set the parameters of this estimator. The method works on simple estimators as well as on nested objects (such as pipelines). The latter have parameters of the form <component>__<parameter> so that it's possible to update each component of a nested object. Returns self adapted from https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/base.py Author: Gael Varoquaux <gael.varoquaux@normalesup.org> License: BSD 3 clause","title":"Methods"},{"location":"api_modules/mlxtend.classifier/SoftmaxRegression/","text":"SoftmaxRegression SoftmaxRegression(eta=0.01, epochs=50, l2=0.0, minibatches=1, n_classes=None, random_seed=None, print_progress=0) Softmax regression classifier. Parameters eta : float (default: 0.01) Learning rate (between 0.0 and 1.0) epochs : int (default: 50) Passes over the training dataset. Prior to each epoch, the dataset is shuffled if minibatches > 1 to prevent cycles in stochastic gradient descent. l2 : float Regularization parameter for L2 regularization. No regularization if l2=0.0. minibatches : int (default: 1) The number of minibatches for gradient-based optimization. If 1: Gradient Descent learning If len(y): Stochastic Gradient Descent (SGD) online learning If 1 < minibatches < len(y): SGD Minibatch learning n_classes : int (default: None) A positive integer to declare the number of class labels if not all class labels are present in a partial training set. Gets the number of class labels automatically if None. random_seed : int (default: None) Set random state for shuffling and initializing the weights. print_progress : int (default: 0) Prints progress in fitting to stderr. 0: No output 1: Epochs elapsed and cost 2: 1 plus time elapsed 3: 2 plus estimated time until completion Attributes w_ : 2d-array, shape={n_features, 1} Model weights after fitting. b_ : 1d-array, shape={1,} Bias unit after fitting. cost_ : list List of floats, the average cross_entropy for each epoch. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/classifier/SoftmaxRegression/ Methods fit(X, y, init_params=True) Learn model from training data. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values. init_params : bool (default: True) Re-initializes model parameters prior to fitting. Set False to continue training with weights from a previous model fitting. Returns self : object get_params(deep=True) Get parameters for this estimator. Parameters deep : boolean, optional If True, will return the parameters for this estimator and contained subobjects that are estimators. Returns params : mapping of string to any Parameter names mapped to their values.' adapted from https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/base.py Author: Gael Varoquaux gael.varoquaux@normalesup.org License: BSD 3 clause predict(X) Predict targets from X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns target_values : array-like, shape = [n_samples] Predicted target values. predict_proba(X) Predict class probabilities of X from the net input. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns Class probabilties : array-like, shape= [n_samples, n_classes] score(X, y) Compute the prediction accuracy Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values (true class labels). Returns acc : float The prediction accuracy as a float between 0.0 and 1.0 (perfect score). set_params( params) Set the parameters of this estimator. The method works on simple estimators as well as on nested objects (such as pipelines). The latter have parameters of the form <component>__<parameter> so that it's possible to update each component of a nested object. Returns self adapted from https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/base.py Author: Gael Varoquaux <gael.varoquaux@normalesup.org> License: BSD 3 clause","title":"SoftmaxRegression"},{"location":"api_modules/mlxtend.classifier/SoftmaxRegression/#softmaxregression","text":"SoftmaxRegression(eta=0.01, epochs=50, l2=0.0, minibatches=1, n_classes=None, random_seed=None, print_progress=0) Softmax regression classifier. Parameters eta : float (default: 0.01) Learning rate (between 0.0 and 1.0) epochs : int (default: 50) Passes over the training dataset. Prior to each epoch, the dataset is shuffled if minibatches > 1 to prevent cycles in stochastic gradient descent. l2 : float Regularization parameter for L2 regularization. No regularization if l2=0.0. minibatches : int (default: 1) The number of minibatches for gradient-based optimization. If 1: Gradient Descent learning If len(y): Stochastic Gradient Descent (SGD) online learning If 1 < minibatches < len(y): SGD Minibatch learning n_classes : int (default: None) A positive integer to declare the number of class labels if not all class labels are present in a partial training set. Gets the number of class labels automatically if None. random_seed : int (default: None) Set random state for shuffling and initializing the weights. print_progress : int (default: 0) Prints progress in fitting to stderr. 0: No output 1: Epochs elapsed and cost 2: 1 plus time elapsed 3: 2 plus estimated time until completion Attributes w_ : 2d-array, shape={n_features, 1} Model weights after fitting. b_ : 1d-array, shape={1,} Bias unit after fitting. cost_ : list List of floats, the average cross_entropy for each epoch. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/classifier/SoftmaxRegression/","title":"SoftmaxRegression"},{"location":"api_modules/mlxtend.classifier/SoftmaxRegression/#methods","text":"fit(X, y, init_params=True) Learn model from training data. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values. init_params : bool (default: True) Re-initializes model parameters prior to fitting. Set False to continue training with weights from a previous model fitting. Returns self : object get_params(deep=True) Get parameters for this estimator. Parameters deep : boolean, optional If True, will return the parameters for this estimator and contained subobjects that are estimators. Returns params : mapping of string to any Parameter names mapped to their values.' adapted from https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/base.py Author: Gael Varoquaux gael.varoquaux@normalesup.org License: BSD 3 clause predict(X) Predict targets from X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns target_values : array-like, shape = [n_samples] Predicted target values. predict_proba(X) Predict class probabilities of X from the net input. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns Class probabilties : array-like, shape= [n_samples, n_classes] score(X, y) Compute the prediction accuracy Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values (true class labels). Returns acc : float The prediction accuracy as a float between 0.0 and 1.0 (perfect score). set_params( params) Set the parameters of this estimator. The method works on simple estimators as well as on nested objects (such as pipelines). The latter have parameters of the form <component>__<parameter> so that it's possible to update each component of a nested object. Returns self adapted from https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/base.py Author: Gael Varoquaux <gael.varoquaux@normalesup.org> License: BSD 3 clause","title":"Methods"},{"location":"api_modules/mlxtend.classifier/StackingCVClassifier/","text":"StackingCVClassifier StackingCVClassifier(classifiers, meta_classifier, use_probas=False, drop_proba_col=None, cv=2, shuffle=True, random_state=None, stratify=True, verbose=0, use_features_in_secondary=False, store_train_meta_features=False, use_clones=True, n_jobs=None, pre_dispatch='2 n_jobs')* A 'Stacking Cross-Validation' classifier for scikit-learn estimators. New in mlxtend v0.4.3 Parameters classifiers : array-like, shape = [n_classifiers] A list of classifiers. Invoking the fit method on the StackingCVClassifer will fit clones of these original classifiers that will be stored in the class attribute self.clfs_ if use_clones=True . meta_classifier : object The meta-classifier to be fitted on the ensemble of classifiers use_probas : bool (default: False) If True, trains meta-classifier based on predicted probabilities instead of class labels. drop_proba_col : string (default: None) Drops extra \"probability\" column in the feature set, because it is redundant: p(y_c) = 1 - p(y_1) + p(y_2) + ... + p(y_{c-1}). This can be useful for meta-classifiers that are sensitive to perfectly collinear features. If 'last', drops last probability column. If 'first', drops first probability column. Only relevant if use_probas=True . cv : int, cross-validation generator or an iterable, optional (default: 2) Determines the cross-validation splitting strategy. Possible inputs for cv are: - None, to use the default 2-fold cross validation, - integer, to specify the number of folds in a (Stratified)KFold , - An object to be used as a cross-validation generator. - An iterable yielding train, test splits. For integer/None inputs, it will use either a KFold or StratifiedKFold cross validation depending the value of stratify argument. shuffle : bool (default: True) If True, and the cv argument is integer, the training data will be shuffled at fitting stage prior to cross-validation. If the cv argument is a specific cross validation technique, this argument is omitted. random_state : int, RandomState instance or None, optional (default: None) Constrols the randomness of the cv splitter. Used when cv is integer and shuffle=True . New in v0.16.0. stratify : bool (default: True) If True, and the cv argument is integer it will follow a stratified K-Fold cross validation technique. If the cv argument is a specific cross validation technique, this argument is omitted. verbose : int, optional (default=0) Controls the verbosity of the building process. - verbose=0 (default): Prints nothing - verbose=1 : Prints the number & name of the regressor being fitted and which fold is currently being used for fitting - verbose=2 : Prints info about the parameters of the regressor being fitted - verbose>2 : Changes verbose param of the underlying regressor to self.verbose - 2 use_features_in_secondary : bool (default: False) If True, the meta-classifier will be trained both on the predictions of the original classifiers and the original dataset. If False, the meta-classifier will be trained only on the predictions of the original classifiers. store_train_meta_features : bool (default: False) If True, the meta-features computed from the training data used for fitting the meta-classifier stored in the self.train_meta_features_ array, which can be accessed after calling fit . use_clones : bool (default: True) Clones the classifiers for stacking classification if True (default) or else uses the original ones, which will be refitted on the dataset upon calling the fit method. Hence, if use_clones=True, the original input classifiers will remain unmodified upon using the StackingCVClassifier's fit method. Setting use_clones=False is recommended if you are working with estimators that are supporting the scikit-learn fit/predict API interface but are not compatible to scikit-learn's clone function. n_jobs : int or None, optional (default=None) The number of CPUs to use to do the computation. None means 1 unless in a joblib.parallel_backend context. -1 means using all processors. for more details. New in v0.16.0. pre_dispatch : int, or string, optional Controls the number of jobs that get dispatched during parallel execution. Reducing this number can be useful to avoid an explosion of memory consumption when more jobs get dispatched than CPUs can process. This parameter can be: - None, in which case all the jobs are immediately created and spawned. Use this for lightweight and fast-running jobs, to avoid delays due to on-demand spawning of the jobs - An int, giving the exact number of total jobs that are spawned - A string, giving an expression as a function of n_jobs, as in '2*n_jobs' New in v0.16.0. Attributes clfs_ : list, shape=[n_classifiers] Fitted classifiers (clones of the original classifiers) meta_clf_ : estimator Fitted meta-classifier (clone of the original meta-estimator) train_meta_features : numpy array, shape = [n_samples, n_classifiers] meta-features for training data, where n_samples is the number of samples in training data and n_classifiers is the number of classfiers. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/classifier/StackingCVClassifier/ Methods decision_function(X) Predict class confidence scores for X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns scores : shape=(n_samples,) if n_classes == 2 else (n_samples, n_classes). Confidence scores per (sample, class) combination. In the binary case, confidence score for self.classes_[1] where >0 means this class would be predicted. fit(X, y, groups=None, sample_weight=None) Fit ensemble classifers and the meta-classifier. Parameters X : numpy array, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : numpy array, shape = [n_samples] Target values. groups : numpy array/None, shape = [n_samples] The group that each sample belongs to. This is used by specific folding strategies such as GroupKFold() sample_weight : array-like, shape = [n_samples], optional Sample weights passed as sample_weights to each regressor in the regressors list as well as the meta_regressor. Raises error if some regressor does not support sample_weight in the fit() method. Returns self : object fit_transform(X, y=None, fit_params) Fit to data, then transform it. Fits transformer to `X` and `y` with optional parameters `fit_params` and returns a transformed version of `X`. Parameters X : array-like of shape (n_samples, n_features) Input samples. y : array-like of shape (n_samples,) or (n_samples, n_outputs), default=None Target values (None for unsupervised transformations). **fit_params : dict Additional fit parameters. Returns X_new : ndarray array of shape (n_samples, n_features_new) Transformed array. get_params(deep=True) Return estimator parameter names for GridSearch support. predict(X) Predict target values for X. Parameters X : numpy array, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns labels : array-like, shape = [n_samples] Predicted class labels. predict_meta_features(X) Get meta-features of test-data. Parameters X : numpy array, shape = [n_samples, n_features] Test vectors, where n_samples is the number of samples and n_features is the number of features. Returns meta-features : numpy array, shape = [n_samples, n_classifiers] Returns the meta-features for test data. predict_proba(X) Predict class probabilities for X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns proba : array-like, shape = [n_samples, n_classes] or a list of n_outputs of such arrays if n_outputs > 1. Probability for each class per sample. score(X, y, sample_weight=None) Return the mean accuracy on the given test data and labels. In multi-label classification, this is the subset accuracy which is a harsh metric since you require for each sample that each label set be correctly predicted. Parameters X : array-like of shape (n_samples, n_features) Test samples. y : array-like of shape (n_samples,) or (n_samples, n_outputs) True labels for X . sample_weight : array-like of shape (n_samples,), default=None Sample weights. Returns score : float Mean accuracy of self.predict(X) w.r.t. y . set_output( , transform=None)* Set output container. See :ref:`sphx_glr_auto_examples_miscellaneous_plot_set_output.py` for an example on how to use the API. Parameters transform : {\"default\", \"pandas\"}, default=None Configure output of transform and fit_transform . \"default\" : Default output format of a transformer \"pandas\" : DataFrame output None : Transform configuration is unchanged Returns self : estimator instance Estimator instance. set_params( params) Set the parameters of this estimator. Valid parameter keys can be listed with ``get_params()``. Returns self Properties named_classifiers None","title":"StackingCVClassifier"},{"location":"api_modules/mlxtend.classifier/StackingCVClassifier/#stackingcvclassifier","text":"StackingCVClassifier(classifiers, meta_classifier, use_probas=False, drop_proba_col=None, cv=2, shuffle=True, random_state=None, stratify=True, verbose=0, use_features_in_secondary=False, store_train_meta_features=False, use_clones=True, n_jobs=None, pre_dispatch='2 n_jobs')* A 'Stacking Cross-Validation' classifier for scikit-learn estimators. New in mlxtend v0.4.3 Parameters classifiers : array-like, shape = [n_classifiers] A list of classifiers. Invoking the fit method on the StackingCVClassifer will fit clones of these original classifiers that will be stored in the class attribute self.clfs_ if use_clones=True . meta_classifier : object The meta-classifier to be fitted on the ensemble of classifiers use_probas : bool (default: False) If True, trains meta-classifier based on predicted probabilities instead of class labels. drop_proba_col : string (default: None) Drops extra \"probability\" column in the feature set, because it is redundant: p(y_c) = 1 - p(y_1) + p(y_2) + ... + p(y_{c-1}). This can be useful for meta-classifiers that are sensitive to perfectly collinear features. If 'last', drops last probability column. If 'first', drops first probability column. Only relevant if use_probas=True . cv : int, cross-validation generator or an iterable, optional (default: 2) Determines the cross-validation splitting strategy. Possible inputs for cv are: - None, to use the default 2-fold cross validation, - integer, to specify the number of folds in a (Stratified)KFold , - An object to be used as a cross-validation generator. - An iterable yielding train, test splits. For integer/None inputs, it will use either a KFold or StratifiedKFold cross validation depending the value of stratify argument. shuffle : bool (default: True) If True, and the cv argument is integer, the training data will be shuffled at fitting stage prior to cross-validation. If the cv argument is a specific cross validation technique, this argument is omitted. random_state : int, RandomState instance or None, optional (default: None) Constrols the randomness of the cv splitter. Used when cv is integer and shuffle=True . New in v0.16.0. stratify : bool (default: True) If True, and the cv argument is integer it will follow a stratified K-Fold cross validation technique. If the cv argument is a specific cross validation technique, this argument is omitted. verbose : int, optional (default=0) Controls the verbosity of the building process. - verbose=0 (default): Prints nothing - verbose=1 : Prints the number & name of the regressor being fitted and which fold is currently being used for fitting - verbose=2 : Prints info about the parameters of the regressor being fitted - verbose>2 : Changes verbose param of the underlying regressor to self.verbose - 2 use_features_in_secondary : bool (default: False) If True, the meta-classifier will be trained both on the predictions of the original classifiers and the original dataset. If False, the meta-classifier will be trained only on the predictions of the original classifiers. store_train_meta_features : bool (default: False) If True, the meta-features computed from the training data used for fitting the meta-classifier stored in the self.train_meta_features_ array, which can be accessed after calling fit . use_clones : bool (default: True) Clones the classifiers for stacking classification if True (default) or else uses the original ones, which will be refitted on the dataset upon calling the fit method. Hence, if use_clones=True, the original input classifiers will remain unmodified upon using the StackingCVClassifier's fit method. Setting use_clones=False is recommended if you are working with estimators that are supporting the scikit-learn fit/predict API interface but are not compatible to scikit-learn's clone function. n_jobs : int or None, optional (default=None) The number of CPUs to use to do the computation. None means 1 unless in a joblib.parallel_backend context. -1 means using all processors. for more details. New in v0.16.0. pre_dispatch : int, or string, optional Controls the number of jobs that get dispatched during parallel execution. Reducing this number can be useful to avoid an explosion of memory consumption when more jobs get dispatched than CPUs can process. This parameter can be: - None, in which case all the jobs are immediately created and spawned. Use this for lightweight and fast-running jobs, to avoid delays due to on-demand spawning of the jobs - An int, giving the exact number of total jobs that are spawned - A string, giving an expression as a function of n_jobs, as in '2*n_jobs' New in v0.16.0. Attributes clfs_ : list, shape=[n_classifiers] Fitted classifiers (clones of the original classifiers) meta_clf_ : estimator Fitted meta-classifier (clone of the original meta-estimator) train_meta_features : numpy array, shape = [n_samples, n_classifiers] meta-features for training data, where n_samples is the number of samples in training data and n_classifiers is the number of classfiers. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/classifier/StackingCVClassifier/","title":"StackingCVClassifier"},{"location":"api_modules/mlxtend.classifier/StackingCVClassifier/#methods","text":"decision_function(X) Predict class confidence scores for X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns scores : shape=(n_samples,) if n_classes == 2 else (n_samples, n_classes). Confidence scores per (sample, class) combination. In the binary case, confidence score for self.classes_[1] where >0 means this class would be predicted. fit(X, y, groups=None, sample_weight=None) Fit ensemble classifers and the meta-classifier. Parameters X : numpy array, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : numpy array, shape = [n_samples] Target values. groups : numpy array/None, shape = [n_samples] The group that each sample belongs to. This is used by specific folding strategies such as GroupKFold() sample_weight : array-like, shape = [n_samples], optional Sample weights passed as sample_weights to each regressor in the regressors list as well as the meta_regressor. Raises error if some regressor does not support sample_weight in the fit() method. Returns self : object fit_transform(X, y=None, fit_params) Fit to data, then transform it. Fits transformer to `X` and `y` with optional parameters `fit_params` and returns a transformed version of `X`. Parameters X : array-like of shape (n_samples, n_features) Input samples. y : array-like of shape (n_samples,) or (n_samples, n_outputs), default=None Target values (None for unsupervised transformations). **fit_params : dict Additional fit parameters. Returns X_new : ndarray array of shape (n_samples, n_features_new) Transformed array. get_params(deep=True) Return estimator parameter names for GridSearch support. predict(X) Predict target values for X. Parameters X : numpy array, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns labels : array-like, shape = [n_samples] Predicted class labels. predict_meta_features(X) Get meta-features of test-data. Parameters X : numpy array, shape = [n_samples, n_features] Test vectors, where n_samples is the number of samples and n_features is the number of features. Returns meta-features : numpy array, shape = [n_samples, n_classifiers] Returns the meta-features for test data. predict_proba(X) Predict class probabilities for X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns proba : array-like, shape = [n_samples, n_classes] or a list of n_outputs of such arrays if n_outputs > 1. Probability for each class per sample. score(X, y, sample_weight=None) Return the mean accuracy on the given test data and labels. In multi-label classification, this is the subset accuracy which is a harsh metric since you require for each sample that each label set be correctly predicted. Parameters X : array-like of shape (n_samples, n_features) Test samples. y : array-like of shape (n_samples,) or (n_samples, n_outputs) True labels for X . sample_weight : array-like of shape (n_samples,), default=None Sample weights. Returns score : float Mean accuracy of self.predict(X) w.r.t. y . set_output( , transform=None)* Set output container. See :ref:`sphx_glr_auto_examples_miscellaneous_plot_set_output.py` for an example on how to use the API. Parameters transform : {\"default\", \"pandas\"}, default=None Configure output of transform and fit_transform . \"default\" : Default output format of a transformer \"pandas\" : DataFrame output None : Transform configuration is unchanged Returns self : estimator instance Estimator instance. set_params( params) Set the parameters of this estimator. Valid parameter keys can be listed with ``get_params()``. Returns self","title":"Methods"},{"location":"api_modules/mlxtend.classifier/StackingCVClassifier/#properties","text":"named_classifiers None","title":"Properties"},{"location":"api_modules/mlxtend.classifier/StackingClassifier/","text":"StackingClassifier StackingClassifier(classifiers, meta_classifier, use_probas=False, drop_proba_col=None, average_probas=False, verbose=0, use_features_in_secondary=False, store_train_meta_features=False, use_clones=True, fit_base_estimators=True) A Stacking classifier for scikit-learn estimators for classification. Parameters classifiers : array-like, shape = [n_classifiers] A list of classifiers. Invoking the fit method on the StackingClassifer will fit clones of these original classifiers that will be stored in the class attribute self.clfs_ if use_clones=True (default) and fit_base_estimators=True (default). meta_classifier : object The meta-classifier to be fitted on the ensemble of classifiers use_probas : bool (default: False) If True, trains meta-classifier based on predicted probabilities instead of class labels. drop_proba_col : string (default: None) Drops extra \"probability\" column in the feature set, because it is redundant: p(y_c) = 1 - p(y_1) + p(y_2) + ... + p(y_{c-1}). This can be useful for meta-classifiers that are sensitive to perfectly collinear features. If 'last', drops last probability column. If 'first', drops first probability column. Only relevant if use_probas=True . average_probas : bool (default: False) Averages the probabilities as meta features if True . Only relevant if use_probas=True . verbose : int, optional (default=0) Controls the verbosity of the building process. - verbose=0 (default): Prints nothing - verbose=1 : Prints the number & name of the regressor being fitted - verbose=2 : Prints info about the parameters of the regressor being fitted - verbose>2 : Changes verbose param of the underlying regressor to self.verbose - 2 use_features_in_secondary : bool (default: False) If True, the meta-classifier will be trained both on the predictions of the original classifiers and the original dataset. If False, the meta-classifier will be trained only on the predictions of the original classifiers. store_train_meta_features : bool (default: False) If True, the meta-features computed from the training data used for fitting the meta-classifier stored in the self.train_meta_features_ array, which can be accessed after calling fit . use_clones : bool (default: True) Clones the classifiers for stacking classification if True (default) or else uses the original ones, which will be refitted on the dataset upon calling the fit method. Hence, if use_clones=True, the original input classifiers will remain unmodified upon using the StackingClassifier's fit method. Setting use_clones=False is recommended if you are working with estimators that are supporting the scikit-learn fit/predict API interface but are not compatible to scikit-learn's clone function. fit_base_estimators: bool (default: True) Refits classifiers in classifiers if True; uses references to the classifiers , otherwise (assumes that the classifiers were already fit). Note: fit_base_estimators=False will enforce use_clones to be False, and is incompatible to most scikit-learn wrappers! For instance, if any form of cross-validation is performed this would require the re-fitting classifiers to training folds, which would raise a NotFitterError if fit_base_estimators=False. (New in mlxtend v0.6.) Attributes clfs_ : list, shape=[n_classifiers] Fitted classifiers (clones of the original classifiers) meta_clf_ : estimator Fitted meta-classifier (clone of the original meta-estimator) train_meta_features : numpy array, shape = [n_samples, n_classifiers] meta-features for training data, where n_samples is the number of samples in training data and n_classifiers is the number of classfiers. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/classifier/StackingClassifier/ Methods decision_function(X) Predict class confidence scores for X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns scores : shape=(n_samples,) if n_classes == 2 else (n_samples, n_classes). Confidence scores per (sample, class) combination. In the binary case, confidence score for self.classes_[1] where >0 means this class would be predicted. fit(X, y, sample_weight=None) Fit ensemble classifers and the meta-classifier. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] or [n_samples, n_outputs] Target values. sample_weight : array-like, shape = [n_samples], optional Sample weights passed as sample_weights to each regressor in the regressors list as well as the meta_regressor. Raises error if some regressor does not support sample_weight in the fit() method. Returns self : object fit_transform(X, y=None, fit_params) Fit to data, then transform it. Fits transformer to `X` and `y` with optional parameters `fit_params` and returns a transformed version of `X`. Parameters X : array-like of shape (n_samples, n_features) Input samples. y : array-like of shape (n_samples,) or (n_samples, n_outputs), default=None Target values (None for unsupervised transformations). **fit_params : dict Additional fit parameters. Returns X_new : ndarray array of shape (n_samples, n_features_new) Transformed array. get_params(deep=True) Return estimator parameter names for GridSearch support. predict(X) Predict target values for X. Parameters X : numpy array, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns labels : array-like, shape = [n_samples] Predicted class labels. predict_meta_features(X) Get meta-features of test-data. Parameters X : numpy array, shape = [n_samples, n_features] Test vectors, where n_samples is the number of samples and n_features is the number of features. Returns meta-features : numpy array, shape = [n_samples, n_classifiers] Returns the meta-features for test data. predict_proba(X) Predict class probabilities for X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns proba : array-like, shape = [n_samples, n_classes] or a list of n_outputs of such arrays if n_outputs > 1. Probability for each class per sample. score(X, y, sample_weight=None) Return the mean accuracy on the given test data and labels. In multi-label classification, this is the subset accuracy which is a harsh metric since you require for each sample that each label set be correctly predicted. Parameters X : array-like of shape (n_samples, n_features) Test samples. y : array-like of shape (n_samples,) or (n_samples, n_outputs) True labels for X . sample_weight : array-like of shape (n_samples,), default=None Sample weights. Returns score : float Mean accuracy of self.predict(X) w.r.t. y . set_output( , transform=None)* Set output container. See :ref:`sphx_glr_auto_examples_miscellaneous_plot_set_output.py` for an example on how to use the API. Parameters transform : {\"default\", \"pandas\"}, default=None Configure output of transform and fit_transform . \"default\" : Default output format of a transformer \"pandas\" : DataFrame output None : Transform configuration is unchanged Returns self : estimator instance Estimator instance. set_params( params) Set the parameters of this estimator. Valid parameter keys can be listed with ``get_params()``. Returns self Properties named_classifiers None","title":"StackingClassifier"},{"location":"api_modules/mlxtend.classifier/StackingClassifier/#stackingclassifier","text":"StackingClassifier(classifiers, meta_classifier, use_probas=False, drop_proba_col=None, average_probas=False, verbose=0, use_features_in_secondary=False, store_train_meta_features=False, use_clones=True, fit_base_estimators=True) A Stacking classifier for scikit-learn estimators for classification. Parameters classifiers : array-like, shape = [n_classifiers] A list of classifiers. Invoking the fit method on the StackingClassifer will fit clones of these original classifiers that will be stored in the class attribute self.clfs_ if use_clones=True (default) and fit_base_estimators=True (default). meta_classifier : object The meta-classifier to be fitted on the ensemble of classifiers use_probas : bool (default: False) If True, trains meta-classifier based on predicted probabilities instead of class labels. drop_proba_col : string (default: None) Drops extra \"probability\" column in the feature set, because it is redundant: p(y_c) = 1 - p(y_1) + p(y_2) + ... + p(y_{c-1}). This can be useful for meta-classifiers that are sensitive to perfectly collinear features. If 'last', drops last probability column. If 'first', drops first probability column. Only relevant if use_probas=True . average_probas : bool (default: False) Averages the probabilities as meta features if True . Only relevant if use_probas=True . verbose : int, optional (default=0) Controls the verbosity of the building process. - verbose=0 (default): Prints nothing - verbose=1 : Prints the number & name of the regressor being fitted - verbose=2 : Prints info about the parameters of the regressor being fitted - verbose>2 : Changes verbose param of the underlying regressor to self.verbose - 2 use_features_in_secondary : bool (default: False) If True, the meta-classifier will be trained both on the predictions of the original classifiers and the original dataset. If False, the meta-classifier will be trained only on the predictions of the original classifiers. store_train_meta_features : bool (default: False) If True, the meta-features computed from the training data used for fitting the meta-classifier stored in the self.train_meta_features_ array, which can be accessed after calling fit . use_clones : bool (default: True) Clones the classifiers for stacking classification if True (default) or else uses the original ones, which will be refitted on the dataset upon calling the fit method. Hence, if use_clones=True, the original input classifiers will remain unmodified upon using the StackingClassifier's fit method. Setting use_clones=False is recommended if you are working with estimators that are supporting the scikit-learn fit/predict API interface but are not compatible to scikit-learn's clone function. fit_base_estimators: bool (default: True) Refits classifiers in classifiers if True; uses references to the classifiers , otherwise (assumes that the classifiers were already fit). Note: fit_base_estimators=False will enforce use_clones to be False, and is incompatible to most scikit-learn wrappers! For instance, if any form of cross-validation is performed this would require the re-fitting classifiers to training folds, which would raise a NotFitterError if fit_base_estimators=False. (New in mlxtend v0.6.) Attributes clfs_ : list, shape=[n_classifiers] Fitted classifiers (clones of the original classifiers) meta_clf_ : estimator Fitted meta-classifier (clone of the original meta-estimator) train_meta_features : numpy array, shape = [n_samples, n_classifiers] meta-features for training data, where n_samples is the number of samples in training data and n_classifiers is the number of classfiers. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/classifier/StackingClassifier/","title":"StackingClassifier"},{"location":"api_modules/mlxtend.classifier/StackingClassifier/#methods","text":"decision_function(X) Predict class confidence scores for X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns scores : shape=(n_samples,) if n_classes == 2 else (n_samples, n_classes). Confidence scores per (sample, class) combination. In the binary case, confidence score for self.classes_[1] where >0 means this class would be predicted. fit(X, y, sample_weight=None) Fit ensemble classifers and the meta-classifier. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] or [n_samples, n_outputs] Target values. sample_weight : array-like, shape = [n_samples], optional Sample weights passed as sample_weights to each regressor in the regressors list as well as the meta_regressor. Raises error if some regressor does not support sample_weight in the fit() method. Returns self : object fit_transform(X, y=None, fit_params) Fit to data, then transform it. Fits transformer to `X` and `y` with optional parameters `fit_params` and returns a transformed version of `X`. Parameters X : array-like of shape (n_samples, n_features) Input samples. y : array-like of shape (n_samples,) or (n_samples, n_outputs), default=None Target values (None for unsupervised transformations). **fit_params : dict Additional fit parameters. Returns X_new : ndarray array of shape (n_samples, n_features_new) Transformed array. get_params(deep=True) Return estimator parameter names for GridSearch support. predict(X) Predict target values for X. Parameters X : numpy array, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns labels : array-like, shape = [n_samples] Predicted class labels. predict_meta_features(X) Get meta-features of test-data. Parameters X : numpy array, shape = [n_samples, n_features] Test vectors, where n_samples is the number of samples and n_features is the number of features. Returns meta-features : numpy array, shape = [n_samples, n_classifiers] Returns the meta-features for test data. predict_proba(X) Predict class probabilities for X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns proba : array-like, shape = [n_samples, n_classes] or a list of n_outputs of such arrays if n_outputs > 1. Probability for each class per sample. score(X, y, sample_weight=None) Return the mean accuracy on the given test data and labels. In multi-label classification, this is the subset accuracy which is a harsh metric since you require for each sample that each label set be correctly predicted. Parameters X : array-like of shape (n_samples, n_features) Test samples. y : array-like of shape (n_samples,) or (n_samples, n_outputs) True labels for X . sample_weight : array-like of shape (n_samples,), default=None Sample weights. Returns score : float Mean accuracy of self.predict(X) w.r.t. y . set_output( , transform=None)* Set output container. See :ref:`sphx_glr_auto_examples_miscellaneous_plot_set_output.py` for an example on how to use the API. Parameters transform : {\"default\", \"pandas\"}, default=None Configure output of transform and fit_transform . \"default\" : Default output format of a transformer \"pandas\" : DataFrame output None : Transform configuration is unchanged Returns self : estimator instance Estimator instance. set_params( params) Set the parameters of this estimator. Valid parameter keys can be listed with ``get_params()``. Returns self","title":"Methods"},{"location":"api_modules/mlxtend.classifier/StackingClassifier/#properties","text":"named_classifiers None","title":"Properties"},{"location":"api_modules/mlxtend.cluster/Kmeans/","text":"Kmeans Kmeans(k, max_iter=10, convergence_tolerance=1e-05, random_seed=None, print_progress=0) K-means clustering class. Added in 0.4.1dev Parameters k : int Number of clusters max_iter : int (default: 10) Number of iterations during cluster assignment. Cluster re-assignment stops automatically when the algorithm converged. convergence_tolerance : float (default: 1e-05) Compares current centroids with centroids of the previous iteration using the given tolerance (a small positive float)to determine if the algorithm converged early. random_seed : int (default: None) Set random state for the initial centroid assignment. print_progress : int (default: 0) Prints progress in fitting to stderr. 0: No output 1: Iterations elapsed 2: 1 plus time elapsed 3: 2 plus estimated time until completion Attributes centroids_ : 2d-array, shape={k, n_features} Feature values of the k cluster centroids. custers_ : dictionary The cluster assignments stored as a Python dictionary; the dictionary keys denote the cluster indeces and the items are Python lists of the sample indices that were assigned to each cluster. iterations_ : int Number of iterations until convergence. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/classifier/Kmeans/ Methods fit(X, init_params=True) Learn model from training data. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. init_params : bool (default: True) Re-initializes model parameters prior to fitting. Set False to continue training with weights from a previous model fitting. Returns self : object get_params(deep=True) Get parameters for this estimator. Parameters deep : boolean, optional If True, will return the parameters for this estimator and contained subobjects that are estimators. Returns params : mapping of string to any Parameter names mapped to their values.' adapted from https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/base.py Author: Gael Varoquaux gael.varoquaux@normalesup.org License: BSD 3 clause predict(X) Predict targets from X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns target_values : array-like, shape = [n_samples] Predicted target values. set_params( params) Set the parameters of this estimator. The method works on simple estimators as well as on nested objects (such as pipelines). The latter have parameters of the form <component>__<parameter> so that it's possible to update each component of a nested object. Returns self adapted from https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/base.py Author: Gael Varoquaux <gael.varoquaux@normalesup.org> License: BSD 3 clause","title":"Kmeans"},{"location":"api_modules/mlxtend.cluster/Kmeans/#kmeans","text":"Kmeans(k, max_iter=10, convergence_tolerance=1e-05, random_seed=None, print_progress=0) K-means clustering class. Added in 0.4.1dev Parameters k : int Number of clusters max_iter : int (default: 10) Number of iterations during cluster assignment. Cluster re-assignment stops automatically when the algorithm converged. convergence_tolerance : float (default: 1e-05) Compares current centroids with centroids of the previous iteration using the given tolerance (a small positive float)to determine if the algorithm converged early. random_seed : int (default: None) Set random state for the initial centroid assignment. print_progress : int (default: 0) Prints progress in fitting to stderr. 0: No output 1: Iterations elapsed 2: 1 plus time elapsed 3: 2 plus estimated time until completion Attributes centroids_ : 2d-array, shape={k, n_features} Feature values of the k cluster centroids. custers_ : dictionary The cluster assignments stored as a Python dictionary; the dictionary keys denote the cluster indeces and the items are Python lists of the sample indices that were assigned to each cluster. iterations_ : int Number of iterations until convergence. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/classifier/Kmeans/","title":"Kmeans"},{"location":"api_modules/mlxtend.cluster/Kmeans/#methods","text":"fit(X, init_params=True) Learn model from training data. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. init_params : bool (default: True) Re-initializes model parameters prior to fitting. Set False to continue training with weights from a previous model fitting. Returns self : object get_params(deep=True) Get parameters for this estimator. Parameters deep : boolean, optional If True, will return the parameters for this estimator and contained subobjects that are estimators. Returns params : mapping of string to any Parameter names mapped to their values.' adapted from https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/base.py Author: Gael Varoquaux gael.varoquaux@normalesup.org License: BSD 3 clause predict(X) Predict targets from X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns target_values : array-like, shape = [n_samples] Predicted target values. set_params( params) Set the parameters of this estimator. The method works on simple estimators as well as on nested objects (such as pipelines). The latter have parameters of the form <component>__<parameter> so that it's possible to update each component of a nested object. Returns self adapted from https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/base.py Author: Gael Varoquaux <gael.varoquaux@normalesup.org> License: BSD 3 clause","title":"Methods"},{"location":"api_modules/mlxtend.data/autompg_data/","text":"autompg_data autompg_data() Auto MPG dataset. Source : https://archive.ics.uci.edu/ml/datasets/Auto+MPG Number of samples : 392 Continuous target variable : mpg Dataset Attributes: 1) cylinders: multi-valued discrete 2) displacement: continuous 3) horsepower: continuous 4) weight: continuous 5) acceleration: continuous 6) model year: multi-valued discrete 7) origin: multi-valued discrete 8) car name: string (unique for each instance) Returns X, y : [n_samples, n_features], [n_targets] X is the feature matrix with 392 auto samples as rows and 8 feature columns (6 rows with NaNs removed). y is a 1-dimensional array of the target MPG values. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/data/autompg_data/","title":"Autompg data"},{"location":"api_modules/mlxtend.data/autompg_data/#autompg_data","text":"autompg_data() Auto MPG dataset. Source : https://archive.ics.uci.edu/ml/datasets/Auto+MPG Number of samples : 392 Continuous target variable : mpg Dataset Attributes: 1) cylinders: multi-valued discrete 2) displacement: continuous 3) horsepower: continuous 4) weight: continuous 5) acceleration: continuous 6) model year: multi-valued discrete 7) origin: multi-valued discrete 8) car name: string (unique for each instance) Returns X, y : [n_samples, n_features], [n_targets] X is the feature matrix with 392 auto samples as rows and 8 feature columns (6 rows with NaNs removed). y is a 1-dimensional array of the target MPG values. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/data/autompg_data/","title":"autompg_data"},{"location":"api_modules/mlxtend.data/boston_housing_data/","text":"boston_housing_data boston_housing_data() Boston Housing dataset. Source : https://archive.ics.uci.edu/ml/datasets/Housing Number of samples : 506 Continuous target variable : MEDV MEDV = Median value of owner-occupied homes in $1000's Dataset Attributes: 1) CRIM per capita crime rate by town 2) ZN proportion of residential land zoned for lots over 25,000 sq.ft. 3) INDUS proportion of non-retail business acres per town 4) CHAS Charles River dummy variable (= 1 if tract bounds river; 0 otherwise) 5) NOX nitric oxides concentration (parts per 10 million) 6) RM average number of rooms per dwelling 7) AGE proportion of owner-occupied units built prior to 1940 8) DIS weighted distances to five Boston employment centres 9) RAD index of accessibility to radial highways 10) TAX full-value property-tax rate per $10,000 11) PTRATIO pupil-teacher ratio by town 12) B 1000(Bk - 0.63)^2 where Bk is the prop. of b. by town 13) LSTAT % lower status of the population Returns X, y : [n_samples, n_features], [n_class_labels] X is the feature matrix with 506 housing samples as rows and 13 feature columns. y is a 1-dimensional array of the continuous target variable MEDV Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/data/boston_housing_data/","title":"Boston housing data"},{"location":"api_modules/mlxtend.data/boston_housing_data/#boston_housing_data","text":"boston_housing_data() Boston Housing dataset. Source : https://archive.ics.uci.edu/ml/datasets/Housing Number of samples : 506 Continuous target variable : MEDV MEDV = Median value of owner-occupied homes in $1000's Dataset Attributes: 1) CRIM per capita crime rate by town 2) ZN proportion of residential land zoned for lots over 25,000 sq.ft. 3) INDUS proportion of non-retail business acres per town 4) CHAS Charles River dummy variable (= 1 if tract bounds river; 0 otherwise) 5) NOX nitric oxides concentration (parts per 10 million) 6) RM average number of rooms per dwelling 7) AGE proportion of owner-occupied units built prior to 1940 8) DIS weighted distances to five Boston employment centres 9) RAD index of accessibility to radial highways 10) TAX full-value property-tax rate per $10,000 11) PTRATIO pupil-teacher ratio by town 12) B 1000(Bk - 0.63)^2 where Bk is the prop. of b. by town 13) LSTAT % lower status of the population Returns X, y : [n_samples, n_features], [n_class_labels] X is the feature matrix with 506 housing samples as rows and 13 feature columns. y is a 1-dimensional array of the continuous target variable MEDV Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/data/boston_housing_data/","title":"boston_housing_data"},{"location":"api_modules/mlxtend.data/iris_data/","text":"iris_data iris_data(version='uci') Iris flower dataset. Source : https://archive.ics.uci.edu/ml/datasets/Iris Number of samples : 150 Class labels : {0, 1, 2}, distribution: [50, 50, 50] 0 = setosa, 1 = versicolor, 2 = virginica. Dataset Attributes: 1) sepal length [cm] 2) sepal width [cm] 3) petal length [cm] 4) petal width [cm] Parameters version : string, optional (default: 'uci'). Version to use {'uci', 'corrected'}. 'uci' loads the dataset as deposited on the UCI machine learning repository, and 'corrected' provides the version that is consistent with Fisher's original paper. See Note for details. Returns X, y : [n_samples, n_features], [n_class_labels] X is the feature matrix with 150 flower samples as rows, and 4 feature columns sepal length, sepal width, petal length, and petal width. y is a 1-dimensional array of the class labels {0, 1, 2} Note The Iris dataset (originally collected by Edgar Anderson) and available in UCI's machine learning repository is different from the Iris dataset described in the original paper by R.A. Fisher [1]). Precisely, there are two data points (row number 34 and 37) in UCI's Machine Learning repository are different from the origianlly published Iris dataset. Also, the original version of the Iris Dataset, which can be loaded via version='corrected' is the same as the one in R. [1] . A. Fisher (1936). \"The use of multiple measurements in taxonomic problems\". Annals of Eugenics. 7 (2): 179\u2013188 Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/data/iris_data/","title":"Iris data"},{"location":"api_modules/mlxtend.data/iris_data/#iris_data","text":"iris_data(version='uci') Iris flower dataset. Source : https://archive.ics.uci.edu/ml/datasets/Iris Number of samples : 150 Class labels : {0, 1, 2}, distribution: [50, 50, 50] 0 = setosa, 1 = versicolor, 2 = virginica. Dataset Attributes: 1) sepal length [cm] 2) sepal width [cm] 3) petal length [cm] 4) petal width [cm] Parameters version : string, optional (default: 'uci'). Version to use {'uci', 'corrected'}. 'uci' loads the dataset as deposited on the UCI machine learning repository, and 'corrected' provides the version that is consistent with Fisher's original paper. See Note for details. Returns X, y : [n_samples, n_features], [n_class_labels] X is the feature matrix with 150 flower samples as rows, and 4 feature columns sepal length, sepal width, petal length, and petal width. y is a 1-dimensional array of the class labels {0, 1, 2} Note The Iris dataset (originally collected by Edgar Anderson) and available in UCI's machine learning repository is different from the Iris dataset described in the original paper by R.A. Fisher [1]). Precisely, there are two data points (row number 34 and 37) in UCI's Machine Learning repository are different from the origianlly published Iris dataset. Also, the original version of the Iris Dataset, which can be loaded via version='corrected' is the same as the one in R. [1] . A. Fisher (1936). \"The use of multiple measurements in taxonomic problems\". Annals of Eugenics. 7 (2): 179\u2013188 Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/data/iris_data/","title":"iris_data"},{"location":"api_modules/mlxtend.data/loadlocal_mnist/","text":"loadlocal_mnist loadlocal_mnist(images_path, labels_path) Read MNIST from ubyte files. Parameters images_path : str path to the test or train MNIST ubyte file labels_path : str path to the test or train MNIST class labels file Returns images : [n_samples, n_pixels] numpy.array Pixel values of the images. labels : [n_samples] numpy array Target class labels Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/data/loadlocal_mnist/","title":"Loadlocal mnist"},{"location":"api_modules/mlxtend.data/loadlocal_mnist/#loadlocal_mnist","text":"loadlocal_mnist(images_path, labels_path) Read MNIST from ubyte files. Parameters images_path : str path to the test or train MNIST ubyte file labels_path : str path to the test or train MNIST class labels file Returns images : [n_samples, n_pixels] numpy.array Pixel values of the images. labels : [n_samples] numpy array Target class labels Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/data/loadlocal_mnist/","title":"loadlocal_mnist"},{"location":"api_modules/mlxtend.data/make_multiplexer_dataset/","text":"make_multiplexer_dataset make_multiplexer_dataset(address_bits=2, sample_size=100, positive_class_ratio=0.5, shuffle=False, random_seed=None) Function to create a binary n-bit multiplexer dataset. New in mlxtend v0.9 Parameters address_bits : int (default: 2) A positive integer that determines the number of address bits in the multiplexer, which in turn determine the n-bit capacity of the multiplexer and therefore the number of features. The number of features is determined by the number of address bits. For example, 2 address bits will result in a 6 bit multiplexer and consequently 6 features (2 + 2^2 = 6). If address_bits=3 , then this results in an 11-bit multiplexer as (2 + 2^3 = 11) with 11 features. sample_size : int (default: 100) The total number of samples generated. positive_class_ratio : float (default: 0.5) The fraction (a float between 0 and 1) of samples in the sample_size d dataset that have class label 1. If positive_class_ratio=0.5 (default), then the ratio of class 0 and class 1 samples is perfectly balanced. shuffle : Bool (default: False) Whether or not to shuffle the features and labels. If False (default), the samples are returned in sorted order starting with sample_size /2 samples with class label 0 and followed by sample_size /2 samples with class label 1. random_seed : int (default: None) Random seed used for generating the multiplexer samples and shuffling. Returns X, y : [n_samples, n_features], [n_class_labels] X is the feature matrix with the number of samples equal to sample_size . The number of features is determined by the number of address bits. For instance, 2 address bits will result in a 6 bit multiplexer and consequently 6 features (2 + 2^2 = 6). All features are binary (values in {0, 1}). y is a 1-dimensional array of class labels in {0, 1}. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/data/make_multiplexer_dataset","title":"Make multiplexer dataset"},{"location":"api_modules/mlxtend.data/make_multiplexer_dataset/#make_multiplexer_dataset","text":"make_multiplexer_dataset(address_bits=2, sample_size=100, positive_class_ratio=0.5, shuffle=False, random_seed=None) Function to create a binary n-bit multiplexer dataset. New in mlxtend v0.9 Parameters address_bits : int (default: 2) A positive integer that determines the number of address bits in the multiplexer, which in turn determine the n-bit capacity of the multiplexer and therefore the number of features. The number of features is determined by the number of address bits. For example, 2 address bits will result in a 6 bit multiplexer and consequently 6 features (2 + 2^2 = 6). If address_bits=3 , then this results in an 11-bit multiplexer as (2 + 2^3 = 11) with 11 features. sample_size : int (default: 100) The total number of samples generated. positive_class_ratio : float (default: 0.5) The fraction (a float between 0 and 1) of samples in the sample_size d dataset that have class label 1. If positive_class_ratio=0.5 (default), then the ratio of class 0 and class 1 samples is perfectly balanced. shuffle : Bool (default: False) Whether or not to shuffle the features and labels. If False (default), the samples are returned in sorted order starting with sample_size /2 samples with class label 0 and followed by sample_size /2 samples with class label 1. random_seed : int (default: None) Random seed used for generating the multiplexer samples and shuffling. Returns X, y : [n_samples, n_features], [n_class_labels] X is the feature matrix with the number of samples equal to sample_size . The number of features is determined by the number of address bits. For instance, 2 address bits will result in a 6 bit multiplexer and consequently 6 features (2 + 2^2 = 6). All features are binary (values in {0, 1}). y is a 1-dimensional array of class labels in {0, 1}. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/data/make_multiplexer_dataset","title":"make_multiplexer_dataset"},{"location":"api_modules/mlxtend.data/mnist_data/","text":"mnist_data mnist_data() 5000 samples from the MNIST handwritten digits dataset. Data Source : https://yann.lecun.com/exdb/mnist/ Returns X, y : [n_samples, n_features], [n_class_labels] X is the feature matrix with 5000 image samples as rows, each row consists of 28x28 pixels that were unrolled into 784 pixel feature vectors. y contains the 10 unique class labels 0-9. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/data/mnist_data/","title":"Mnist data"},{"location":"api_modules/mlxtend.data/mnist_data/#mnist_data","text":"mnist_data() 5000 samples from the MNIST handwritten digits dataset. Data Source : https://yann.lecun.com/exdb/mnist/ Returns X, y : [n_samples, n_features], [n_class_labels] X is the feature matrix with 5000 image samples as rows, each row consists of 28x28 pixels that were unrolled into 784 pixel feature vectors. y contains the 10 unique class labels 0-9. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/data/mnist_data/","title":"mnist_data"},{"location":"api_modules/mlxtend.data/three_blobs_data/","text":"three_blobs_data three_blobs_data() A random dataset of 3 2D blobs for clustering. Number of samples : 150 Suggested labels : {0, 1, 2}, distribution: [50, 50, 50] Returns X, y : [n_samples, n_features], [n_cluster_labels] X is the feature matrix with 159 samples as rows and 2 feature columns. y is a 1-dimensional array of the 3 suggested cluster labels 0, 1, 2 Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/data/three_blobs_data","title":"Three blobs data"},{"location":"api_modules/mlxtend.data/three_blobs_data/#three_blobs_data","text":"three_blobs_data() A random dataset of 3 2D blobs for clustering. Number of samples : 150 Suggested labels : {0, 1, 2}, distribution: [50, 50, 50] Returns X, y : [n_samples, n_features], [n_cluster_labels] X is the feature matrix with 159 samples as rows and 2 feature columns. y is a 1-dimensional array of the 3 suggested cluster labels 0, 1, 2 Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/data/three_blobs_data","title":"three_blobs_data"},{"location":"api_modules/mlxtend.data/wine_data/","text":"wine_data wine_data() Wine dataset. Source : https://archive.ics.uci.edu/ml/datasets/Wine Number of samples : 178 Class labels : {0, 1, 2}, distribution: [59, 71, 48] Dataset Attributes: 1) Alcohol 2) Malic acid 3) Ash 4) Alcalinity of ash 5) Magnesium 6) Total phenols 7) Flavanoids 8) Nonflavanoid phenols 9) Proanthocyanins 10) Color intensity 11) Hue 12) OD280/OD315 of diluted wines 13) Proline Returns X, y : [n_samples, n_features], [n_class_labels] X is the feature matrix with 178 wine samples as rows and 13 feature columns. y is a 1-dimensional array of the 3 class labels 0, 1, 2 Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/data/wine_data","title":"Wine data"},{"location":"api_modules/mlxtend.data/wine_data/#wine_data","text":"wine_data() Wine dataset. Source : https://archive.ics.uci.edu/ml/datasets/Wine Number of samples : 178 Class labels : {0, 1, 2}, distribution: [59, 71, 48] Dataset Attributes: 1) Alcohol 2) Malic acid 3) Ash 4) Alcalinity of ash 5) Magnesium 6) Total phenols 7) Flavanoids 8) Nonflavanoid phenols 9) Proanthocyanins 10) Color intensity 11) Hue 12) OD280/OD315 of diluted wines 13) Proline Returns X, y : [n_samples, n_features], [n_class_labels] X is the feature matrix with 178 wine samples as rows and 13 feature columns. y is a 1-dimensional array of the 3 class labels 0, 1, 2 Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/data/wine_data","title":"wine_data"},{"location":"api_modules/mlxtend.evaluate/BootstrapOutOfBag/","text":"BootstrapOutOfBag BootstrapOutOfBag(n_splits=200, random_seed=None) Parameters n_splits : int (default=200) Number of bootstrap iterations. Must be larger than 1. random_seed : int (default=None) If int, random_seed is the seed used by the random number generator. Returns train_idx : ndarray The training set indices for that split. test_idx : ndarray The testing set indices for that split. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/BootstrapOutOfBag/ Methods get_n_splits(X=None, y=None, groups=None) Returns the number of splitting iterations in the cross-validator Parameters X : object Always ignored, exists for compatibility with scikit-learn. y : object Always ignored, exists for compatibility with scikit-learn. groups : object Always ignored, exists for compatibility with scikit-learn. Returns n_splits : int Returns the number of splitting iterations in the cross-validator. split(X, y=None, groups=None) y : array-like or None (default: None) Argument is not used and only included as parameter for compatibility, similar to KFold in scikit-learn. groups : array-like or None (default: None) Argument is not used and only included as parameter for compatibility, similar to KFold in scikit-learn.","title":"BootstrapOutOfBag"},{"location":"api_modules/mlxtend.evaluate/BootstrapOutOfBag/#bootstrapoutofbag","text":"BootstrapOutOfBag(n_splits=200, random_seed=None) Parameters n_splits : int (default=200) Number of bootstrap iterations. Must be larger than 1. random_seed : int (default=None) If int, random_seed is the seed used by the random number generator. Returns train_idx : ndarray The training set indices for that split. test_idx : ndarray The testing set indices for that split. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/BootstrapOutOfBag/","title":"BootstrapOutOfBag"},{"location":"api_modules/mlxtend.evaluate/BootstrapOutOfBag/#methods","text":"get_n_splits(X=None, y=None, groups=None) Returns the number of splitting iterations in the cross-validator Parameters X : object Always ignored, exists for compatibility with scikit-learn. y : object Always ignored, exists for compatibility with scikit-learn. groups : object Always ignored, exists for compatibility with scikit-learn. Returns n_splits : int Returns the number of splitting iterations in the cross-validator. split(X, y=None, groups=None) y : array-like or None (default: None) Argument is not used and only included as parameter for compatibility, similar to KFold in scikit-learn. groups : array-like or None (default: None) Argument is not used and only included as parameter for compatibility, similar to KFold in scikit-learn.","title":"Methods"},{"location":"api_modules/mlxtend.evaluate/GroupTimeSeriesSplit/","text":"GroupTimeSeriesSplit GroupTimeSeriesSplit(test_size, train_size=None, n_splits=None, gap_size=0, shift_size=1, window_type='rolling') Group time series cross-validator. Parameters test_size : int Size of test dataset. train_size : int (default=None) Size of train dataset. n_splits : int (default=None) Number of the splits. gap_size : int (default=0) Gap size between train and test datasets. shift_size : int (default=1) Step to shift for the next fold. window_type : str (default=\"rolling\") Type of the window. Possible values: \"rolling\", \"expanding\". Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/GroupTimeSeriesSplit/ Methods get_n_splits(X=None, y=None, groups=None) Returns the number of splitting iterations in the cross-validator. Parameters X : object Always ignored, exists for compatibility. y : object Always ignored, exists for compatibility. groups : object Always ignored, exists for compatibility. Returns n_splits : int Returns the number of splitting iterations in the cross-validator. split(X, y=None, groups=None) Generate indices to split data into training and test set. Parameters X : array-like Training data. y : array-like (default=None) Always ignored, exists for compatibility. groups : array-like (default=None) Array with group names or sequence numbers. Yields train : ndarray The training set indices for that split. test : ndarray The testing set indices for that split.","title":"GroupTimeSeriesSplit"},{"location":"api_modules/mlxtend.evaluate/GroupTimeSeriesSplit/#grouptimeseriessplit","text":"GroupTimeSeriesSplit(test_size, train_size=None, n_splits=None, gap_size=0, shift_size=1, window_type='rolling') Group time series cross-validator. Parameters test_size : int Size of test dataset. train_size : int (default=None) Size of train dataset. n_splits : int (default=None) Number of the splits. gap_size : int (default=0) Gap size between train and test datasets. shift_size : int (default=1) Step to shift for the next fold. window_type : str (default=\"rolling\") Type of the window. Possible values: \"rolling\", \"expanding\". Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/GroupTimeSeriesSplit/","title":"GroupTimeSeriesSplit"},{"location":"api_modules/mlxtend.evaluate/GroupTimeSeriesSplit/#methods","text":"get_n_splits(X=None, y=None, groups=None) Returns the number of splitting iterations in the cross-validator. Parameters X : object Always ignored, exists for compatibility. y : object Always ignored, exists for compatibility. groups : object Always ignored, exists for compatibility. Returns n_splits : int Returns the number of splitting iterations in the cross-validator. split(X, y=None, groups=None) Generate indices to split data into training and test set. Parameters X : array-like Training data. y : array-like (default=None) Always ignored, exists for compatibility. groups : array-like (default=None) Array with group names or sequence numbers. Yields train : ndarray The training set indices for that split. test : ndarray The testing set indices for that split.","title":"Methods"},{"location":"api_modules/mlxtend.evaluate/PredefinedHoldoutSplit/","text":"PredefinedHoldoutSplit PredefinedHoldoutSplit(valid_indices) Train/Validation set splitter for sklearn's GridSearchCV etc. Uses user-specified train/validation set indices to split a dataset into train/validation sets using user-defined or random indices. Parameters valid_indices : array-like, shape (num_examples,) Indices of the training examples in the training set to be used for validation. All other indices in the training set are used to for a training subset for model fitting. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/PredefinedHoldoutSplit/ Methods get_n_splits(X=None, y=None, groups=None) Returns the number of splitting iterations in the cross-validator Parameters X : object Always ignored, exists for compatibility. y : object Always ignored, exists for compatibility. groups : object Always ignored, exists for compatibility. Returns n_splits : 1 Returns the number of splitting iterations in the cross-validator. Always returns 1. split(X, y, groups=None) Generate indices to split data into training and test set. Parameters X : array-like, shape (num_examples, num_features) Training data, where num_examples is the number of examples and num_features is the number of features. y : array-like, shape (num_examples,) The target variable for supervised learning problems. Stratification is done based on the y labels. groups : object Always ignored, exists for compatibility. Yields train_index : ndarray The training set indices for that split. valid_index : ndarray The validation set indices for that split.","title":"PredefinedHoldoutSplit"},{"location":"api_modules/mlxtend.evaluate/PredefinedHoldoutSplit/#predefinedholdoutsplit","text":"PredefinedHoldoutSplit(valid_indices) Train/Validation set splitter for sklearn's GridSearchCV etc. Uses user-specified train/validation set indices to split a dataset into train/validation sets using user-defined or random indices. Parameters valid_indices : array-like, shape (num_examples,) Indices of the training examples in the training set to be used for validation. All other indices in the training set are used to for a training subset for model fitting. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/PredefinedHoldoutSplit/","title":"PredefinedHoldoutSplit"},{"location":"api_modules/mlxtend.evaluate/PredefinedHoldoutSplit/#methods","text":"get_n_splits(X=None, y=None, groups=None) Returns the number of splitting iterations in the cross-validator Parameters X : object Always ignored, exists for compatibility. y : object Always ignored, exists for compatibility. groups : object Always ignored, exists for compatibility. Returns n_splits : 1 Returns the number of splitting iterations in the cross-validator. Always returns 1. split(X, y, groups=None) Generate indices to split data into training and test set. Parameters X : array-like, shape (num_examples, num_features) Training data, where num_examples is the number of examples and num_features is the number of features. y : array-like, shape (num_examples,) The target variable for supervised learning problems. Stratification is done based on the y labels. groups : object Always ignored, exists for compatibility. Yields train_index : ndarray The training set indices for that split. valid_index : ndarray The validation set indices for that split.","title":"Methods"},{"location":"api_modules/mlxtend.evaluate/RandomHoldoutSplit/","text":"RandomHoldoutSplit RandomHoldoutSplit(valid_size=0.5, random_seed=None, stratify=False) Train/Validation set splitter for sklearn's GridSearchCV etc. Provides train/validation set indices to split a dataset into train/validation sets using random indices. Parameters valid_size : float (default: 0.5) Proportion of examples that being assigned as validation examples. 1- valid_size will then automatically be assigned as training set examples. random_seed : int (default: None) The random seed for splitting the data into training and validation set partitions. stratify : bool (default: False) True or False, whether to perform a stratified split or not Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/RandomHoldoutSplit/ Methods get_n_splits(X=None, y=None, groups=None) Returns the number of splitting iterations in the cross-validator Parameters X : object Always ignored, exists for compatibility. y : object Always ignored, exists for compatibility. groups : object Always ignored, exists for compatibility. Returns n_splits : 1 Returns the number of splitting iterations in the cross-validator. Always returns 1. split(X, y, groups=None) Generate indices to split data into training and test set. Parameters X : array-like, shape (num_examples, num_features) Training data, where num_examples is the number of training examples and num_features is the number of features. y : array-like, shape (num_examples,) The target variable for supervised learning problems. Stratification is done based on the y labels. groups : object Always ignored, exists for compatibility. Yields train_index : ndarray The training set indices for that split. valid_index : ndarray The validation set indices for that split.","title":"RandomHoldoutSplit"},{"location":"api_modules/mlxtend.evaluate/RandomHoldoutSplit/#randomholdoutsplit","text":"RandomHoldoutSplit(valid_size=0.5, random_seed=None, stratify=False) Train/Validation set splitter for sklearn's GridSearchCV etc. Provides train/validation set indices to split a dataset into train/validation sets using random indices. Parameters valid_size : float (default: 0.5) Proportion of examples that being assigned as validation examples. 1- valid_size will then automatically be assigned as training set examples. random_seed : int (default: None) The random seed for splitting the data into training and validation set partitions. stratify : bool (default: False) True or False, whether to perform a stratified split or not Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/RandomHoldoutSplit/","title":"RandomHoldoutSplit"},{"location":"api_modules/mlxtend.evaluate/RandomHoldoutSplit/#methods","text":"get_n_splits(X=None, y=None, groups=None) Returns the number of splitting iterations in the cross-validator Parameters X : object Always ignored, exists for compatibility. y : object Always ignored, exists for compatibility. groups : object Always ignored, exists for compatibility. Returns n_splits : 1 Returns the number of splitting iterations in the cross-validator. Always returns 1. split(X, y, groups=None) Generate indices to split data into training and test set. Parameters X : array-like, shape (num_examples, num_features) Training data, where num_examples is the number of training examples and num_features is the number of features. y : array-like, shape (num_examples,) The target variable for supervised learning problems. Stratification is done based on the y labels. groups : object Always ignored, exists for compatibility. Yields train_index : ndarray The training set indices for that split. valid_index : ndarray The validation set indices for that split.","title":"Methods"},{"location":"api_modules/mlxtend.evaluate/accuracy_score/","text":"accuracy_score accuracy_score(y_target, y_predicted, method='standard', pos_label=1, normalize=True) General accuracy function for supervised learning. Parameters y_target : array-like, shape=[n_values] True class labels or target values. y_predicted : array-like, shape=[n_values] Predicted class labels or target values. method : str, 'standard' by default. The chosen method for accuracy computation. If set to 'standard', computes overall accuracy. If set to 'binary', computes accuracy for class pos_label. If set to 'average', computes average per-class (balanced) accuracy. If set to 'balanced', computes the scikit-learn-style balanced accuracy. pos_label : str or int, 1 by default. The class whose accuracy score is to be reported. Used only when method is set to 'binary' normalize : bool, True by default. If True, returns fraction of correctly classified samples. If False, returns number of correctly classified samples. Returns score: float Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/accuracy_score/","title":"Accuracy score"},{"location":"api_modules/mlxtend.evaluate/accuracy_score/#accuracy_score","text":"accuracy_score(y_target, y_predicted, method='standard', pos_label=1, normalize=True) General accuracy function for supervised learning. Parameters y_target : array-like, shape=[n_values] True class labels or target values. y_predicted : array-like, shape=[n_values] Predicted class labels or target values. method : str, 'standard' by default. The chosen method for accuracy computation. If set to 'standard', computes overall accuracy. If set to 'binary', computes accuracy for class pos_label. If set to 'average', computes average per-class (balanced) accuracy. If set to 'balanced', computes the scikit-learn-style balanced accuracy. pos_label : str or int, 1 by default. The class whose accuracy score is to be reported. Used only when method is set to 'binary' normalize : bool, True by default. If True, returns fraction of correctly classified samples. If False, returns number of correctly classified samples. Returns score: float Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/accuracy_score/","title":"accuracy_score"},{"location":"api_modules/mlxtend.evaluate/bias_variance_decomp/","text":"bias_variance_decomp bias_variance_decomp(estimator, X_train, y_train, X_test, y_test, loss='0-1_loss', num_rounds=200, random_seed=None, fit_params) estimator : object A classifier or regressor object or class implementing both a fit and predict method similar to the scikit-learn API. X_train : array-like, shape=(num_examples, num_features) A training dataset for drawing the bootstrap samples to carry out the bias-variance decomposition. y_train : array-like, shape=(num_examples) Targets (class labels, continuous values in case of regression) associated with the X_train examples. X_test : array-like, shape=(num_examples, num_features) The test dataset for computing the average loss, bias, and variance. y_test : array-like, shape=(num_examples) Targets (class labels, continuous values in case of regression) associated with the X_test examples. loss : str (default='0-1_loss') Loss function for performing the bias-variance decomposition. Currently allowed values are '0-1_loss' and 'mse'. num_rounds : int (default=200) Number of bootstrap rounds (sampling from the training set) for performing the bias-variance decomposition. Each bootstrap sample has the same size as the original training set. random_seed : int (default=None) Random seed for the bootstrap sampling used for the bias-variance decomposition. fit_params : additional parameters Additional parameters to be passed to the .fit() function of the estimator when it is fit to the bootstrap samples. Returns avg_expected_loss, avg_bias, avg_var : returns the average expected average bias, and average bias (all floats), where the average is computed over the data points in the test set. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/bias_variance_decomp/","title":"Bias variance decomp"},{"location":"api_modules/mlxtend.evaluate/bias_variance_decomp/#bias_variance_decomp","text":"bias_variance_decomp(estimator, X_train, y_train, X_test, y_test, loss='0-1_loss', num_rounds=200, random_seed=None, fit_params) estimator : object A classifier or regressor object or class implementing both a fit and predict method similar to the scikit-learn API. X_train : array-like, shape=(num_examples, num_features) A training dataset for drawing the bootstrap samples to carry out the bias-variance decomposition. y_train : array-like, shape=(num_examples) Targets (class labels, continuous values in case of regression) associated with the X_train examples. X_test : array-like, shape=(num_examples, num_features) The test dataset for computing the average loss, bias, and variance. y_test : array-like, shape=(num_examples) Targets (class labels, continuous values in case of regression) associated with the X_test examples. loss : str (default='0-1_loss') Loss function for performing the bias-variance decomposition. Currently allowed values are '0-1_loss' and 'mse'. num_rounds : int (default=200) Number of bootstrap rounds (sampling from the training set) for performing the bias-variance decomposition. Each bootstrap sample has the same size as the original training set. random_seed : int (default=None) Random seed for the bootstrap sampling used for the bias-variance decomposition. fit_params : additional parameters Additional parameters to be passed to the .fit() function of the estimator when it is fit to the bootstrap samples. Returns avg_expected_loss, avg_bias, avg_var : returns the average expected average bias, and average bias (all floats), where the average is computed over the data points in the test set. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/bias_variance_decomp/","title":"bias_variance_decomp"},{"location":"api_modules/mlxtend.evaluate/bootstrap/","text":"bootstrap bootstrap(x, func, num_rounds=1000, ci=0.95, ddof=1, seed=None) Implements the ordinary nonparametric bootstrap Parameters x : NumPy array, shape=(n_samples, [n_columns]) An one or multidimensional array of data records func : A function which computes a statistic that is used to compute the bootstrap replicates (the statistic computed from the bootstrap samples). This function must return a scalar value. For example, np.mean or np.median would be an acceptable argument for func if x is a 1-dimensional array or vector. num_rounds : int (default=1000) The number of bootstrap samples to draw where each bootstrap sample has the same number of records as the original dataset. ci : int (default=0.95) An integer in the range (0, 1) that represents the confidence level for computing the confidence interval. For example, ci=0.95 (default) will compute the 95% confidence interval from the bootstrap replicates. ddof : int The delta degrees of freedom used when computing the standard error. seed : int or None (default=None) Random seed for generating bootstrap samples. Returns original, standard_error, (lower_ci, upper_ci) : tuple Returns the statistic of the original sample ( original ), the standard error of the estimate, and the respective confidence interval bounds. Examples ``` >>> from mlxtend.evaluate import bootstrap >>> rng = np.random.RandomState(123) >>> x = rng.normal(loc=5., size=100) >>> original, std_err, ci_bounds = bootstrap(x, ... num_rounds=1000, ... func=np.mean, ... ci=0.95, ... seed=123) >>> print('Mean: %.2f, SE: +/- %.2f, CI95: [%.2f, %.2f]' % (original, ... std_err, ... ci_bounds[0], ... ci_bounds[1])) Mean: 5.03, SE: +/- 0.11, CI95: [4.80, 5.26] >>> For more usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/bootstrap/ ```","title":"Bootstrap"},{"location":"api_modules/mlxtend.evaluate/bootstrap/#bootstrap","text":"bootstrap(x, func, num_rounds=1000, ci=0.95, ddof=1, seed=None) Implements the ordinary nonparametric bootstrap Parameters x : NumPy array, shape=(n_samples, [n_columns]) An one or multidimensional array of data records func : A function which computes a statistic that is used to compute the bootstrap replicates (the statistic computed from the bootstrap samples). This function must return a scalar value. For example, np.mean or np.median would be an acceptable argument for func if x is a 1-dimensional array or vector. num_rounds : int (default=1000) The number of bootstrap samples to draw where each bootstrap sample has the same number of records as the original dataset. ci : int (default=0.95) An integer in the range (0, 1) that represents the confidence level for computing the confidence interval. For example, ci=0.95 (default) will compute the 95% confidence interval from the bootstrap replicates. ddof : int The delta degrees of freedom used when computing the standard error. seed : int or None (default=None) Random seed for generating bootstrap samples. Returns original, standard_error, (lower_ci, upper_ci) : tuple Returns the statistic of the original sample ( original ), the standard error of the estimate, and the respective confidence interval bounds. Examples ``` >>> from mlxtend.evaluate import bootstrap >>> rng = np.random.RandomState(123) >>> x = rng.normal(loc=5., size=100) >>> original, std_err, ci_bounds = bootstrap(x, ... num_rounds=1000, ... func=np.mean, ... ci=0.95, ... seed=123) >>> print('Mean: %.2f, SE: +/- %.2f, CI95: [%.2f, %.2f]' % (original, ... std_err, ... ci_bounds[0], ... ci_bounds[1])) Mean: 5.03, SE: +/- 0.11, CI95: [4.80, 5.26] >>> For more usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/bootstrap/ ```","title":"bootstrap"},{"location":"api_modules/mlxtend.evaluate/bootstrap_point632_score/","text":"bootstrap_point632_score bootstrap_point632_score(estimator, X, y, n_splits=200, method='.632', scoring_func=None, predict_proba=False, random_seed=None, clone_estimator=True, fit_params) Implementation of the .632 [1] and .632+ [2] bootstrap for supervised learning References: - [1] Efron, Bradley. 1983. \"Estimating the Error Rate of a Prediction Rule: Improvement on Cross-Validation.\" Journal of the American Statistical Association 78 (382): 316. doi:10.2307/2288636. - [2] Efron, Bradley, and Robert Tibshirani. 1997. \"Improvements on Cross-Validation: The .632+ Bootstrap Method.\" Journal of the American Statistical Association 92 (438): 548. doi:10.2307/2965703. Parameters estimator : object An estimator for classification or regression that follows the scikit-learn API and implements \"fit\" and \"predict\" methods. X : array-like The data to fit. Can be, for example a list, or an array at least 2d. y : array-like, optional, default: None The target variable to try to predict in the case of supervised learning. n_splits : int (default=200) Number of bootstrap iterations. Must be larger than 1. method : str (default='.632') The bootstrap method, which can be either - 1) '.632' bootstrap (default) - 2) '.632+' bootstrap - 3) 'oob' (regular out-of-bag, no weighting) for comparison studies. scoring_func : callable, Score function (or loss function) with signature scoring_func(y, y_pred, **kwargs) . If none, uses classification accuracy if the estimator is a classifier and mean squared error if the estimator is a regressor. predict_proba : bool Whether to use the predict_proba function for the estimator argument. This is to be used in conjunction with scoring_func which takes in probability values instead of actual predictions. For example, if the scoring_func is :meth: sklearn.metrics.roc_auc_score , then use predict_proba=True . Note that this requires estimator to have predict_proba method implemented. random_seed : int (default=None) If int, random_seed is the seed used by the random number generator. clone_estimator : bool (default=True) Clones the estimator if true, otherwise fits the original. fit_params : additional parameters Additional parameters to be passed to the .fit() function of the estimator when it is fit to the bootstrap samples. Returns scores : array of float, shape=(len(list(n_splits)),) Array of scores of the estimator for each bootstrap replicate. Examples >>> from sklearn import datasets, linear_model >>> from mlxtend.evaluate import bootstrap_point632_score >>> iris = datasets.load_iris() >>> X = iris.data >>> y = iris.target >>> lr = linear_model.LogisticRegression() >>> scores = bootstrap_point632_score(lr, X, y) >>> acc = np.mean(scores) >>> print('Accuracy:', acc) 0.953023146884 >>> lower = np.percentile(scores, 2.5) >>> upper = np.percentile(scores, 97.5) >>> print('95%% Confidence interval: [%.2f, %.2f]' % (lower, upper)) 95% Confidence interval: [0.90, 0.98] For more usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/bootstrap_point632_score/","title":"Bootstrap point632 score"},{"location":"api_modules/mlxtend.evaluate/bootstrap_point632_score/#bootstrap_point632_score","text":"bootstrap_point632_score(estimator, X, y, n_splits=200, method='.632', scoring_func=None, predict_proba=False, random_seed=None, clone_estimator=True, fit_params) Implementation of the .632 [1] and .632+ [2] bootstrap for supervised learning References: - [1] Efron, Bradley. 1983. \"Estimating the Error Rate of a Prediction Rule: Improvement on Cross-Validation.\" Journal of the American Statistical Association 78 (382): 316. doi:10.2307/2288636. - [2] Efron, Bradley, and Robert Tibshirani. 1997. \"Improvements on Cross-Validation: The .632+ Bootstrap Method.\" Journal of the American Statistical Association 92 (438): 548. doi:10.2307/2965703. Parameters estimator : object An estimator for classification or regression that follows the scikit-learn API and implements \"fit\" and \"predict\" methods. X : array-like The data to fit. Can be, for example a list, or an array at least 2d. y : array-like, optional, default: None The target variable to try to predict in the case of supervised learning. n_splits : int (default=200) Number of bootstrap iterations. Must be larger than 1. method : str (default='.632') The bootstrap method, which can be either - 1) '.632' bootstrap (default) - 2) '.632+' bootstrap - 3) 'oob' (regular out-of-bag, no weighting) for comparison studies. scoring_func : callable, Score function (or loss function) with signature scoring_func(y, y_pred, **kwargs) . If none, uses classification accuracy if the estimator is a classifier and mean squared error if the estimator is a regressor. predict_proba : bool Whether to use the predict_proba function for the estimator argument. This is to be used in conjunction with scoring_func which takes in probability values instead of actual predictions. For example, if the scoring_func is :meth: sklearn.metrics.roc_auc_score , then use predict_proba=True . Note that this requires estimator to have predict_proba method implemented. random_seed : int (default=None) If int, random_seed is the seed used by the random number generator. clone_estimator : bool (default=True) Clones the estimator if true, otherwise fits the original. fit_params : additional parameters Additional parameters to be passed to the .fit() function of the estimator when it is fit to the bootstrap samples. Returns scores : array of float, shape=(len(list(n_splits)),) Array of scores of the estimator for each bootstrap replicate. Examples >>> from sklearn import datasets, linear_model >>> from mlxtend.evaluate import bootstrap_point632_score >>> iris = datasets.load_iris() >>> X = iris.data >>> y = iris.target >>> lr = linear_model.LogisticRegression() >>> scores = bootstrap_point632_score(lr, X, y) >>> acc = np.mean(scores) >>> print('Accuracy:', acc) 0.953023146884 >>> lower = np.percentile(scores, 2.5) >>> upper = np.percentile(scores, 97.5) >>> print('95%% Confidence interval: [%.2f, %.2f]' % (lower, upper)) 95% Confidence interval: [0.90, 0.98] For more usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/bootstrap_point632_score/","title":"bootstrap_point632_score"},{"location":"api_modules/mlxtend.evaluate/cochrans_q/","text":"cochrans_q cochrans_q(y_target, y_model_predictions)* Cochran's Q test to compare 2 or more models. Parameters y_target : array-like, shape=[n_samples] True class labels as 1D NumPy array. *y_model_predictions : array-likes, shape=[n_samples] Variable number of 2 or more arrays that contain the predicted class labels from models as 1D NumPy array. Returns q, p : float or None, float Returns the Q (chi-squared) value and the p-value Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/cochrans_q/","title":"Cochrans q"},{"location":"api_modules/mlxtend.evaluate/cochrans_q/#cochrans_q","text":"cochrans_q(y_target, y_model_predictions)* Cochran's Q test to compare 2 or more models. Parameters y_target : array-like, shape=[n_samples] True class labels as 1D NumPy array. *y_model_predictions : array-likes, shape=[n_samples] Variable number of 2 or more arrays that contain the predicted class labels from models as 1D NumPy array. Returns q, p : float or None, float Returns the Q (chi-squared) value and the p-value Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/cochrans_q/","title":"cochrans_q"},{"location":"api_modules/mlxtend.evaluate/combined_ftest_5x2cv/","text":"combined_ftest_5x2cv combined_ftest_5x2cv(estimator1, estimator2, X, y, scoring=None, random_seed=None) Implements the 5x2cv combined F test proposed by Alpaydin 1999, to compare the performance of two models. Parameters estimator1 : scikit-learn classifier or regressor estimator2 : scikit-learn classifier or regressor X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values. scoring : str, callable, or None (default: None) If None (default), uses 'accuracy' for sklearn classifiers and 'r2' for sklearn regressors. If str, uses a sklearn scoring metric string identifier, for example {accuracy, f1, precision, recall, roc_auc} for classifiers, {'mean_absolute_error', 'mean_squared_error'/'neg_mean_squared_error', 'median_absolute_error', 'r2'} for regressors. If a callable object or function is provided, it has to be conform with sklearn's signature scorer(estimator, X, y) ; see https://scikit-learn.org/stable/modules/generated/sklearn.metrics.make_scorer.html for more information. random_seed : int or None (default: None) Random seed for creating the test/train splits. Returns f : float The F-statistic pvalue : float Two-tailed p-value. If the chosen significance level is larger than the p-value, we reject the null hypothesis and accept that there are significant differences in the two compared models. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/combined_ftest_5x2cv/","title":"Combined ftest 5x2cv"},{"location":"api_modules/mlxtend.evaluate/combined_ftest_5x2cv/#combined_ftest_5x2cv","text":"combined_ftest_5x2cv(estimator1, estimator2, X, y, scoring=None, random_seed=None) Implements the 5x2cv combined F test proposed by Alpaydin 1999, to compare the performance of two models. Parameters estimator1 : scikit-learn classifier or regressor estimator2 : scikit-learn classifier or regressor X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values. scoring : str, callable, or None (default: None) If None (default), uses 'accuracy' for sklearn classifiers and 'r2' for sklearn regressors. If str, uses a sklearn scoring metric string identifier, for example {accuracy, f1, precision, recall, roc_auc} for classifiers, {'mean_absolute_error', 'mean_squared_error'/'neg_mean_squared_error', 'median_absolute_error', 'r2'} for regressors. If a callable object or function is provided, it has to be conform with sklearn's signature scorer(estimator, X, y) ; see https://scikit-learn.org/stable/modules/generated/sklearn.metrics.make_scorer.html for more information. random_seed : int or None (default: None) Random seed for creating the test/train splits. Returns f : float The F-statistic pvalue : float Two-tailed p-value. If the chosen significance level is larger than the p-value, we reject the null hypothesis and accept that there are significant differences in the two compared models. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/combined_ftest_5x2cv/","title":"combined_ftest_5x2cv"},{"location":"api_modules/mlxtend.evaluate/confusion_matrix/","text":"confusion_matrix confusion_matrix(y_target, y_predicted, binary=False, positive_label=1) Compute a confusion matrix/contingency table. Parameters y_target : array-like, shape=[n_samples] True class labels. y_predicted : array-like, shape=[n_samples] Predicted class labels. binary : bool (default: False) Maps a multi-class problem onto a binary confusion matrix, where the positive class is 1 and all other classes are 0. positive_label : int (default: 1) Class label of the positive class. Returns mat : array-like, shape=[n_classes, n_classes] Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/confusion_matrix/","title":"Confusion matrix"},{"location":"api_modules/mlxtend.evaluate/confusion_matrix/#confusion_matrix","text":"confusion_matrix(y_target, y_predicted, binary=False, positive_label=1) Compute a confusion matrix/contingency table. Parameters y_target : array-like, shape=[n_samples] True class labels. y_predicted : array-like, shape=[n_samples] Predicted class labels. binary : bool (default: False) Maps a multi-class problem onto a binary confusion matrix, where the positive class is 1 and all other classes are 0. positive_label : int (default: 1) Class label of the positive class. Returns mat : array-like, shape=[n_classes, n_classes] Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/confusion_matrix/","title":"confusion_matrix"},{"location":"api_modules/mlxtend.evaluate/create_counterfactual/","text":"create_counterfactual create_counterfactual(x_reference, y_desired, model, X_dataset, y_desired_proba=None, lammbda=0.1, random_seed=None) Implementation of the counterfactual method by Wachter et al. 2017 References: - Wachter, S., Mittelstadt, B., & Russell, C. (2017). Counterfactual explanations without opening the black box: Automated decisions and the GDPR. Harv. JL & Tech., 31, 841., https://arxiv.org/abs/1711.00399 Parameters x_reference : array-like, shape=[m_features] The data instance (training example) to be explained. y_desired : int The desired class label for x_reference . model : estimator A (scikit-learn) estimator implementing .predict() and/or predict_proba() . - If model supports predict_proba() , then this is used by default for the first loss term, (lambda * model.predict[_proba](x_counterfact) - y_desired[_proba])^2 - Otherwise, method will fall back to predict . X_dataset : array-like, shape=[n_examples, m_features] A (training) dataset for picking the initial counterfactual as initial value for starting the optimization procedure. y_desired_proba : float (default: None) A float within the range [0, 1] designating the desired class probability for y_desired . - If y_desired_proba=None (default), the first loss term is (lambda * model(x_counterfact) - y_desired)^2 where y_desired is a class label - If y_desired_proba is not None, the first loss term is (lambda * model(x_counterfact) - y_desired_proba)^2 lammbda : Weighting parameter for the first loss term, (lambda * model(x_counterfact) - y_desired[_proba])^2 random_seed : int (default=None) If int, random_seed is the seed used by the random number generator for selecting the inital counterfactual from X_dataset .","title":"Create counterfactual"},{"location":"api_modules/mlxtend.evaluate/create_counterfactual/#create_counterfactual","text":"create_counterfactual(x_reference, y_desired, model, X_dataset, y_desired_proba=None, lammbda=0.1, random_seed=None) Implementation of the counterfactual method by Wachter et al. 2017 References: - Wachter, S., Mittelstadt, B., & Russell, C. (2017). Counterfactual explanations without opening the black box: Automated decisions and the GDPR. Harv. JL & Tech., 31, 841., https://arxiv.org/abs/1711.00399 Parameters x_reference : array-like, shape=[m_features] The data instance (training example) to be explained. y_desired : int The desired class label for x_reference . model : estimator A (scikit-learn) estimator implementing .predict() and/or predict_proba() . - If model supports predict_proba() , then this is used by default for the first loss term, (lambda * model.predict[_proba](x_counterfact) - y_desired[_proba])^2 - Otherwise, method will fall back to predict . X_dataset : array-like, shape=[n_examples, m_features] A (training) dataset for picking the initial counterfactual as initial value for starting the optimization procedure. y_desired_proba : float (default: None) A float within the range [0, 1] designating the desired class probability for y_desired . - If y_desired_proba=None (default), the first loss term is (lambda * model(x_counterfact) - y_desired)^2 where y_desired is a class label - If y_desired_proba is not None, the first loss term is (lambda * model(x_counterfact) - y_desired_proba)^2 lammbda : Weighting parameter for the first loss term, (lambda * model(x_counterfact) - y_desired[_proba])^2 random_seed : int (default=None) If int, random_seed is the seed used by the random number generator for selecting the inital counterfactual from X_dataset .","title":"create_counterfactual"},{"location":"api_modules/mlxtend.evaluate/feature_importance_permutation/","text":"feature_importance_permutation feature_importance_permutation(X, y, predict_method, metric, num_rounds=1, feature_groups=None, seed=None) Feature importance imputation via permutation importance Parameters X : NumPy array, shape = [n_samples, n_features] Dataset, where n_samples is the number of samples and n_features is the number of features. y : NumPy array, shape = [n_samples] Target values. predict_method : prediction function A callable function that predicts the target values from X. metric : str, callable The metric for evaluating the feature importance through permutation. By default, the strings 'accuracy' is recommended for classifiers and the string 'r2' is recommended for regressors. Optionally, a custom scoring function (e.g., metric=scoring_func ) that accepts two arguments, y_true and y_pred, which have similar shape to the y array. num_rounds : int (default=1) Number of rounds the feature columns are permuted to compute the permutation importance. feature_groups : list or None (default=None) Optional argument for treating certain features as a group. For example [1, 2, [3, 4, 5]] , which can be useful for interpretability, for example, if features 3, 4, 5 are one-hot encoded features. seed : int or None (default=None) Random seed for permuting the feature columns. Returns mean_importance_vals, all_importance_vals : NumPy arrays. The first array, mean_importance_vals has shape [n_features, ] and contains the importance values for all features. The shape of the second array is [n_features, num_rounds] and contains the feature importance for each repetition. If num_rounds=1, it contains the same values as the first array, mean_importance_vals. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/feature_importance_permutation/","title":"Feature importance permutation"},{"location":"api_modules/mlxtend.evaluate/feature_importance_permutation/#feature_importance_permutation","text":"feature_importance_permutation(X, y, predict_method, metric, num_rounds=1, feature_groups=None, seed=None) Feature importance imputation via permutation importance Parameters X : NumPy array, shape = [n_samples, n_features] Dataset, where n_samples is the number of samples and n_features is the number of features. y : NumPy array, shape = [n_samples] Target values. predict_method : prediction function A callable function that predicts the target values from X. metric : str, callable The metric for evaluating the feature importance through permutation. By default, the strings 'accuracy' is recommended for classifiers and the string 'r2' is recommended for regressors. Optionally, a custom scoring function (e.g., metric=scoring_func ) that accepts two arguments, y_true and y_pred, which have similar shape to the y array. num_rounds : int (default=1) Number of rounds the feature columns are permuted to compute the permutation importance. feature_groups : list or None (default=None) Optional argument for treating certain features as a group. For example [1, 2, [3, 4, 5]] , which can be useful for interpretability, for example, if features 3, 4, 5 are one-hot encoded features. seed : int or None (default=None) Random seed for permuting the feature columns. Returns mean_importance_vals, all_importance_vals : NumPy arrays. The first array, mean_importance_vals has shape [n_features, ] and contains the importance values for all features. The shape of the second array is [n_features, num_rounds] and contains the feature importance for each repetition. If num_rounds=1, it contains the same values as the first array, mean_importance_vals. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/feature_importance_permutation/","title":"feature_importance_permutation"},{"location":"api_modules/mlxtend.evaluate/ftest/","text":"ftest ftest(y_target, y_model_predictions)* F-Test test to compare 2 or more models. Parameters y_target : array-like, shape=[n_samples] True class labels as 1D NumPy array. *y_model_predictions : array-likes, shape=[n_samples] Variable number of 2 or more arrays that contain the predicted class labels from models as 1D NumPy array. Returns f, p : float or None, float Returns the F-value and the p-value Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/ftest/","title":"Ftest"},{"location":"api_modules/mlxtend.evaluate/ftest/#ftest","text":"ftest(y_target, y_model_predictions)* F-Test test to compare 2 or more models. Parameters y_target : array-like, shape=[n_samples] True class labels as 1D NumPy array. *y_model_predictions : array-likes, shape=[n_samples] Variable number of 2 or more arrays that contain the predicted class labels from models as 1D NumPy array. Returns f, p : float or None, float Returns the F-value and the p-value Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/ftest/","title":"ftest"},{"location":"api_modules/mlxtend.evaluate/lift_score/","text":"lift_score lift_score(y_target, y_predicted, binary=True, positive_label=1) Lift measures the degree to which the predictions of a classification model are better than randomly-generated predictions. The in terms of True Positives (TP), True Negatives (TN), False Positives (FP), and False Negatives (FN), the lift score is computed as: [ TP / (TP+FP) ] / [ (TP+FN) / (TP+TN+FP+FN) ] Parameters y_target : array-like, shape=[n_samples] True class labels. y_predicted : array-like, shape=[n_samples] Predicted class labels. binary : bool (default: True) Maps a multi-class problem onto a binary, where the positive class is 1 and all other classes are 0. positive_label : int (default: 0) Class label of the positive class. Returns score : float Lift score in the range [0, infinity] Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/lift_score/","title":"Lift score"},{"location":"api_modules/mlxtend.evaluate/lift_score/#lift_score","text":"lift_score(y_target, y_predicted, binary=True, positive_label=1) Lift measures the degree to which the predictions of a classification model are better than randomly-generated predictions. The in terms of True Positives (TP), True Negatives (TN), False Positives (FP), and False Negatives (FN), the lift score is computed as: [ TP / (TP+FP) ] / [ (TP+FN) / (TP+TN+FP+FN) ] Parameters y_target : array-like, shape=[n_samples] True class labels. y_predicted : array-like, shape=[n_samples] Predicted class labels. binary : bool (default: True) Maps a multi-class problem onto a binary, where the positive class is 1 and all other classes are 0. positive_label : int (default: 0) Class label of the positive class. Returns score : float Lift score in the range [0, infinity] Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/lift_score/","title":"lift_score"},{"location":"api_modules/mlxtend.evaluate/mcnemar/","text":"mcnemar mcnemar(ary, corrected=True, exact=False) McNemar test for paired nominal data Parameters ary : array-like, shape=[2, 2] 2 x 2 contigency table (as returned by evaluate.mcnemar_table), where a: ary[0, 0]: # of samples that both models predicted correctly b: ary[0, 1]: # of samples that model 1 got right and model 2 got wrong c: ary[1, 0]: # of samples that model 2 got right and model 1 got wrong d: aryCell [1, 1]: # of samples that both models predicted incorrectly corrected : array-like, shape=[n_samples] (default: True) Uses Edward's continuity correction for chi-squared if True exact : bool, (default: False) If True , uses an exact binomial test comparing b to a binomial distribution with n = b + c and p = 0.5. It is highly recommended to use exact=True for sample sizes < 25 since chi-squared is not well-approximated by the chi-squared distribution! Returns chi2, p : float or None, float Returns the chi-squared value and the p-value; if exact=True (default: False ), chi2 is None Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/mcnemar/","title":"Mcnemar"},{"location":"api_modules/mlxtend.evaluate/mcnemar/#mcnemar","text":"mcnemar(ary, corrected=True, exact=False) McNemar test for paired nominal data Parameters ary : array-like, shape=[2, 2] 2 x 2 contigency table (as returned by evaluate.mcnemar_table), where a: ary[0, 0]: # of samples that both models predicted correctly b: ary[0, 1]: # of samples that model 1 got right and model 2 got wrong c: ary[1, 0]: # of samples that model 2 got right and model 1 got wrong d: aryCell [1, 1]: # of samples that both models predicted incorrectly corrected : array-like, shape=[n_samples] (default: True) Uses Edward's continuity correction for chi-squared if True exact : bool, (default: False) If True , uses an exact binomial test comparing b to a binomial distribution with n = b + c and p = 0.5. It is highly recommended to use exact=True for sample sizes < 25 since chi-squared is not well-approximated by the chi-squared distribution! Returns chi2, p : float or None, float Returns the chi-squared value and the p-value; if exact=True (default: False ), chi2 is None Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/mcnemar/","title":"mcnemar"},{"location":"api_modules/mlxtend.evaluate/mcnemar_table/","text":"mcnemar_table mcnemar_table(y_target, y_model1, y_model2) Compute a 2x2 contigency table for McNemar's test. Parameters y_target : array-like, shape=[n_samples] True class labels as 1D NumPy array. y_model1 : array-like, shape=[n_samples] Predicted class labels from model as 1D NumPy array. y_model2 : array-like, shape=[n_samples] Predicted class labels from model 2 as 1D NumPy array. Returns tb : array-like, shape=[2, 2] 2x2 contingency table with the following contents: a: tb[0, 0]: # of samples that both models predicted correctly b: tb[0, 1]: # of samples that model 1 got right and model 2 got wrong c: tb[1, 0]: # of samples that model 2 got right and model 1 got wrong d: tb[1, 1]: # of samples that both models predicted incorrectly Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/mcnemar_table/","title":"Mcnemar table"},{"location":"api_modules/mlxtend.evaluate/mcnemar_table/#mcnemar_table","text":"mcnemar_table(y_target, y_model1, y_model2) Compute a 2x2 contigency table for McNemar's test. Parameters y_target : array-like, shape=[n_samples] True class labels as 1D NumPy array. y_model1 : array-like, shape=[n_samples] Predicted class labels from model as 1D NumPy array. y_model2 : array-like, shape=[n_samples] Predicted class labels from model 2 as 1D NumPy array. Returns tb : array-like, shape=[2, 2] 2x2 contingency table with the following contents: a: tb[0, 0]: # of samples that both models predicted correctly b: tb[0, 1]: # of samples that model 1 got right and model 2 got wrong c: tb[1, 0]: # of samples that model 2 got right and model 1 got wrong d: tb[1, 1]: # of samples that both models predicted incorrectly Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/mcnemar_table/","title":"mcnemar_table"},{"location":"api_modules/mlxtend.evaluate/mcnemar_tables/","text":"mcnemar_tables mcnemar_tables(y_target, y_model_predictions)* Compute multiple 2x2 contigency tables for McNemar's test or Cochran's Q test. Parameters y_target : array-like, shape=[n_samples] True class labels as 1D NumPy array. y_model_predictions : array-like, shape=[n_samples] Predicted class labels for a model. Returns tables : dict Dictionary of NumPy arrays with shape=[2, 2]. Each dictionary key names the two models to be compared based on the order the models were passed as *y_model_predictions . The number of dictionary entries is equal to the number of pairwise combinations between the m models, i.e., \"m choose 2.\" For example the following target array (containing the true labels) and 3 models y_true = np.array([0, 0, 0, 0, 0, 1, 1, 1, 1, 1]) y_mod0 = np.array([0, 1, 0, 0, 0, 1, 1, 0, 0, 0]) y_mod1 = np.array([0, 0, 1, 1, 0, 1, 1, 0, 0, 0]) y_mod2 = np.array([0, 1, 1, 1, 0, 1, 0, 0, 0, 0]) would result in the following dictionary: {'model_0 vs model_1': array([[ 4., 1.], [ 2., 3.]]), 'model_0 vs model_2': array([[ 3., 0.], [ 3., 4.]]), 'model_1 vs model_2': array([[ 3., 0.], [ 2., 5.]])} Each array is structured in the following way: tb[0, 0]: # of samples that both models predicted correctly tb[0, 1]: # of samples that model a got right and model b got wrong tb[1, 0]: # of samples that model b got right and model a got wrong tb[1, 1]: # of samples that both models predicted incorrectly Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/mcnemar_tables/","title":"Mcnemar tables"},{"location":"api_modules/mlxtend.evaluate/mcnemar_tables/#mcnemar_tables","text":"mcnemar_tables(y_target, y_model_predictions)* Compute multiple 2x2 contigency tables for McNemar's test or Cochran's Q test. Parameters y_target : array-like, shape=[n_samples] True class labels as 1D NumPy array. y_model_predictions : array-like, shape=[n_samples] Predicted class labels for a model. Returns tables : dict Dictionary of NumPy arrays with shape=[2, 2]. Each dictionary key names the two models to be compared based on the order the models were passed as *y_model_predictions . The number of dictionary entries is equal to the number of pairwise combinations between the m models, i.e., \"m choose 2.\" For example the following target array (containing the true labels) and 3 models y_true = np.array([0, 0, 0, 0, 0, 1, 1, 1, 1, 1]) y_mod0 = np.array([0, 1, 0, 0, 0, 1, 1, 0, 0, 0]) y_mod1 = np.array([0, 0, 1, 1, 0, 1, 1, 0, 0, 0]) y_mod2 = np.array([0, 1, 1, 1, 0, 1, 0, 0, 0, 0]) would result in the following dictionary: {'model_0 vs model_1': array([[ 4., 1.], [ 2., 3.]]), 'model_0 vs model_2': array([[ 3., 0.], [ 3., 4.]]), 'model_1 vs model_2': array([[ 3., 0.], [ 2., 5.]])} Each array is structured in the following way: tb[0, 0]: # of samples that both models predicted correctly tb[0, 1]: # of samples that model a got right and model b got wrong tb[1, 0]: # of samples that model b got right and model a got wrong tb[1, 1]: # of samples that both models predicted incorrectly Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/mcnemar_tables/","title":"mcnemar_tables"},{"location":"api_modules/mlxtend.evaluate/paired_ttest_5x2cv/","text":"paired_ttest_5x2cv paired_ttest_5x2cv(estimator1, estimator2, X, y, scoring=None, random_seed=None) Implements the 5x2cv paired t test proposed by Dieterrich (1998) to compare the performance of two models. Parameters estimator1 : scikit-learn classifier or regressor estimator2 : scikit-learn classifier or regressor X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values. scoring : str, callable, or None (default: None) If None (default), uses 'accuracy' for sklearn classifiers and 'r2' for sklearn regressors. If str, uses a sklearn scoring metric string identifier, for example {accuracy, f1, precision, recall, roc_auc} for classifiers, {'mean_absolute_error', 'mean_squared_error'/'neg_mean_squared_error', 'median_absolute_error', 'r2'} for regressors. If a callable object or function is provided, it has to be conform with sklearn's signature scorer(estimator, X, y) ; see https://scikit-learn.org/stable/modules/generated/sklearn.metrics.make_scorer.html for more information. random_seed : int or None (default: None) Random seed for creating the test/train splits. Returns t : float The t-statistic pvalue : float Two-tailed p-value. If the chosen significance level is larger than the p-value, we reject the null hypothesis and accept that there are significant differences in the two compared models. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/paired_ttest_5x2cv/","title":"Paired ttest 5x2cv"},{"location":"api_modules/mlxtend.evaluate/paired_ttest_5x2cv/#paired_ttest_5x2cv","text":"paired_ttest_5x2cv(estimator1, estimator2, X, y, scoring=None, random_seed=None) Implements the 5x2cv paired t test proposed by Dieterrich (1998) to compare the performance of two models. Parameters estimator1 : scikit-learn classifier or regressor estimator2 : scikit-learn classifier or regressor X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values. scoring : str, callable, or None (default: None) If None (default), uses 'accuracy' for sklearn classifiers and 'r2' for sklearn regressors. If str, uses a sklearn scoring metric string identifier, for example {accuracy, f1, precision, recall, roc_auc} for classifiers, {'mean_absolute_error', 'mean_squared_error'/'neg_mean_squared_error', 'median_absolute_error', 'r2'} for regressors. If a callable object or function is provided, it has to be conform with sklearn's signature scorer(estimator, X, y) ; see https://scikit-learn.org/stable/modules/generated/sklearn.metrics.make_scorer.html for more information. random_seed : int or None (default: None) Random seed for creating the test/train splits. Returns t : float The t-statistic pvalue : float Two-tailed p-value. If the chosen significance level is larger than the p-value, we reject the null hypothesis and accept that there are significant differences in the two compared models. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/paired_ttest_5x2cv/","title":"paired_ttest_5x2cv"},{"location":"api_modules/mlxtend.evaluate/paired_ttest_kfold_cv/","text":"paired_ttest_kfold_cv paired_ttest_kfold_cv(estimator1, estimator2, X, y, cv=10, scoring=None, shuffle=False, random_seed=None) Implements the k-fold paired t test procedure to compare the performance of two models. Parameters estimator1 : scikit-learn classifier or regressor estimator2 : scikit-learn classifier or regressor X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values. cv : int (default: 10) Number of splits and iteration for the cross-validation procedure scoring : str, callable, or None (default: None) If None (default), uses 'accuracy' for sklearn classifiers and 'r2' for sklearn regressors. If str, uses a sklearn scoring metric string identifier, for example {accuracy, f1, precision, recall, roc_auc} for classifiers, {'mean_absolute_error', 'mean_squared_error'/'neg_mean_squared_error', 'median_absolute_error', 'r2'} for regressors. If a callable object or function is provided, it has to be conform with sklearn's signature scorer(estimator, X, y) ; see https://scikit-learn.org/stable/modules/generated/sklearn.metrics.make_scorer.html for more information. shuffle : bool (default: True) Whether to shuffle the dataset for generating the k-fold splits. random_seed : int or None (default: None) Random seed for shuffling the dataset for generating the k-fold splits. Ignored if shuffle=False. Returns t : float The t-statistic pvalue : float Two-tailed p-value. If the chosen significance level is larger than the p-value, we reject the null hypothesis and accept that there are significant differences in the two compared models. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/paired_ttest_kfold_cv/","title":"Paired ttest kfold cv"},{"location":"api_modules/mlxtend.evaluate/paired_ttest_kfold_cv/#paired_ttest_kfold_cv","text":"paired_ttest_kfold_cv(estimator1, estimator2, X, y, cv=10, scoring=None, shuffle=False, random_seed=None) Implements the k-fold paired t test procedure to compare the performance of two models. Parameters estimator1 : scikit-learn classifier or regressor estimator2 : scikit-learn classifier or regressor X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values. cv : int (default: 10) Number of splits and iteration for the cross-validation procedure scoring : str, callable, or None (default: None) If None (default), uses 'accuracy' for sklearn classifiers and 'r2' for sklearn regressors. If str, uses a sklearn scoring metric string identifier, for example {accuracy, f1, precision, recall, roc_auc} for classifiers, {'mean_absolute_error', 'mean_squared_error'/'neg_mean_squared_error', 'median_absolute_error', 'r2'} for regressors. If a callable object or function is provided, it has to be conform with sklearn's signature scorer(estimator, X, y) ; see https://scikit-learn.org/stable/modules/generated/sklearn.metrics.make_scorer.html for more information. shuffle : bool (default: True) Whether to shuffle the dataset for generating the k-fold splits. random_seed : int or None (default: None) Random seed for shuffling the dataset for generating the k-fold splits. Ignored if shuffle=False. Returns t : float The t-statistic pvalue : float Two-tailed p-value. If the chosen significance level is larger than the p-value, we reject the null hypothesis and accept that there are significant differences in the two compared models. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/paired_ttest_kfold_cv/","title":"paired_ttest_kfold_cv"},{"location":"api_modules/mlxtend.evaluate/paired_ttest_resampled/","text":"paired_ttest_resampled paired_ttest_resampled(estimator1, estimator2, X, y, num_rounds=30, test_size=0.3, scoring=None, random_seed=None) Implements the resampled paired t test procedure to compare the performance of two models (also called k-hold-out paired t test). Parameters estimator1 : scikit-learn classifier or regressor estimator2 : scikit-learn classifier or regressor X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values. num_rounds : int (default: 30) Number of resampling iterations (i.e., train/test splits) test_size : float or int (default: 0.3) If float, should be between 0.0 and 1.0 and represent the proportion of the dataset to use as a test set. If int, represents the absolute number of test exsamples. scoring : str, callable, or None (default: None) If None (default), uses 'accuracy' for sklearn classifiers and 'r2' for sklearn regressors. If str, uses a sklearn scoring metric string identifier, for example {accuracy, f1, precision, recall, roc_auc} for classifiers, {'mean_absolute_error', 'mean_squared_error'/'neg_mean_squared_error', 'median_absolute_error', 'r2'} for regressors. If a callable object or function is provided, it has to be conform with sklearn's signature scorer(estimator, X, y) ; see https://scikit-learn.org/stable/modules/generated/sklearn.metrics.make_scorer.html for more information. random_seed : int or None (default: None) Random seed for creating the test/train splits. Returns t : float The t-statistic pvalue : float Two-tailed p-value. If the chosen significance level is larger than the p-value, we reject the null hypothesis and accept that there are significant differences in the two compared models. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/paired_ttest_resampled/","title":"Paired ttest resampled"},{"location":"api_modules/mlxtend.evaluate/paired_ttest_resampled/#paired_ttest_resampled","text":"paired_ttest_resampled(estimator1, estimator2, X, y, num_rounds=30, test_size=0.3, scoring=None, random_seed=None) Implements the resampled paired t test procedure to compare the performance of two models (also called k-hold-out paired t test). Parameters estimator1 : scikit-learn classifier or regressor estimator2 : scikit-learn classifier or regressor X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values. num_rounds : int (default: 30) Number of resampling iterations (i.e., train/test splits) test_size : float or int (default: 0.3) If float, should be between 0.0 and 1.0 and represent the proportion of the dataset to use as a test set. If int, represents the absolute number of test exsamples. scoring : str, callable, or None (default: None) If None (default), uses 'accuracy' for sklearn classifiers and 'r2' for sklearn regressors. If str, uses a sklearn scoring metric string identifier, for example {accuracy, f1, precision, recall, roc_auc} for classifiers, {'mean_absolute_error', 'mean_squared_error'/'neg_mean_squared_error', 'median_absolute_error', 'r2'} for regressors. If a callable object or function is provided, it has to be conform with sklearn's signature scorer(estimator, X, y) ; see https://scikit-learn.org/stable/modules/generated/sklearn.metrics.make_scorer.html for more information. random_seed : int or None (default: None) Random seed for creating the test/train splits. Returns t : float The t-statistic pvalue : float Two-tailed p-value. If the chosen significance level is larger than the p-value, we reject the null hypothesis and accept that there are significant differences in the two compared models. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/paired_ttest_resampled/","title":"paired_ttest_resampled"},{"location":"api_modules/mlxtend.evaluate/permutation_test/","text":"permutation_test permutation_test(x, y, func='x_mean != y_mean', method='exact', num_rounds=1000, seed=None, paired=False) Nonparametric permutation test Parameters x : list or numpy array with shape (n_datapoints,) A list or 1D numpy array of the first sample (e.g., the treatment group). y : list or numpy array with shape (n_datapoints,) A list or 1D numpy array of the second sample (e.g., the control group). func : custom function or str (default: 'x_mean != y_mean') function to compute the statistic for the permutation test. - If 'x_mean != y_mean', uses func=lambda x, y: np.abs(np.mean(x) - np.mean(y))) for a two-sided test. - If 'x_mean > y_mean', uses func=lambda x, y: np.mean(x) - np.mean(y)) for a one-sided test. - If 'x_mean < y_mean', uses func=lambda x, y: np.mean(y) - np.mean(x)) for a one-sided test. method : 'approximate' or 'exact' (default: 'exact') If 'exact' (default), all possible permutations are considered. If 'approximate' the number of drawn samples is given by num_rounds . Note that 'exact' is typically not feasible unless the dataset size is relatively small. paired : bool If True, a paired test is performed by only exchanging each datapoint with its associate. num_rounds : int (default: 1000) The number of permutation samples if method='approximate' . seed : int or None (default: None) The random seed for generating permutation samples if method='approximate' . Returns p-value under the null hypothesis Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/permutation_test/","title":"Permutation test"},{"location":"api_modules/mlxtend.evaluate/permutation_test/#permutation_test","text":"permutation_test(x, y, func='x_mean != y_mean', method='exact', num_rounds=1000, seed=None, paired=False) Nonparametric permutation test Parameters x : list or numpy array with shape (n_datapoints,) A list or 1D numpy array of the first sample (e.g., the treatment group). y : list or numpy array with shape (n_datapoints,) A list or 1D numpy array of the second sample (e.g., the control group). func : custom function or str (default: 'x_mean != y_mean') function to compute the statistic for the permutation test. - If 'x_mean != y_mean', uses func=lambda x, y: np.abs(np.mean(x) - np.mean(y))) for a two-sided test. - If 'x_mean > y_mean', uses func=lambda x, y: np.mean(x) - np.mean(y)) for a one-sided test. - If 'x_mean < y_mean', uses func=lambda x, y: np.mean(y) - np.mean(x)) for a one-sided test. method : 'approximate' or 'exact' (default: 'exact') If 'exact' (default), all possible permutations are considered. If 'approximate' the number of drawn samples is given by num_rounds . Note that 'exact' is typically not feasible unless the dataset size is relatively small. paired : bool If True, a paired test is performed by only exchanging each datapoint with its associate. num_rounds : int (default: 1000) The number of permutation samples if method='approximate' . seed : int or None (default: None) The random seed for generating permutation samples if method='approximate' . Returns p-value under the null hypothesis Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/permutation_test/","title":"permutation_test"},{"location":"api_modules/mlxtend.evaluate/proportion_difference/","text":"proportion_difference proportion_difference(proportion_1, proportion_2, n_1, n_2=None) Computes the test statistic and p-value for a difference of proportions test. Parameters proportion_1 : float The first proportion proportion_2 : float The second proportion n_1 : int The sample size of the first test sample n_2 : int or None (default=None) The sample size of the second test sample. If None , n_1 = n_2 . Returns z, p : float or None, float Returns the z-score and the p-value Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/proportion_difference/","title":"Proportion difference"},{"location":"api_modules/mlxtend.evaluate/proportion_difference/#proportion_difference","text":"proportion_difference(proportion_1, proportion_2, n_1, n_2=None) Computes the test statistic and p-value for a difference of proportions test. Parameters proportion_1 : float The first proportion proportion_2 : float The second proportion n_1 : int The sample size of the first test sample n_2 : int or None (default=None) The sample size of the second test sample. If None , n_1 = n_2 . Returns z, p : float or None, float Returns the z-score and the p-value Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/proportion_difference/","title":"proportion_difference"},{"location":"api_modules/mlxtend.evaluate/scoring/","text":"scoring scoring(y_target, y_predicted, metric='error', positive_label=1, unique_labels='auto') Compute a scoring metric for supervised learning. Parameters y_target : array-like, shape=[n_values] True class labels or target values. y_predicted : array-like, shape=[n_values] Predicted class labels or target values. metric : str (default: 'error') Performance metric: 'accuracy': (TP + TN)/(FP + FN + TP + TN) = 1-ERR 'average per-class accuracy': Average per-class accuracy 'average per-class error': Average per-class error 'balanced per-class accuracy': Average per-class accuracy 'balanced per-class error': Average per-class error 'error': (TP + TN)/(FP+ FN + TP + TN) = 1-ACC 'false_positive_rate': FP/N = FP/(FP + TN) 'true_positive_rate': TP/P = TP/(FN + TP) 'true_negative_rate': TN/N = TN/(FP + TN) 'precision': TP/(TP + FP) 'recall': equal to 'true_positive_rate' 'sensitivity': equal to 'true_positive_rate' or 'recall' 'specificity': equal to 'true_negative_rate' 'f1': 2 * (PRE * REC)/(PRE + REC) 'matthews_corr_coef': (TP TN - FP FN) / (sqrt{(TP + FP)( TP + FN )( TN + FP )( TN + FN )}) Where: [TP: True positives, TN = True negatives, TN: True negatives, FN = False negatives] positive_label : int (default: 1) Label of the positive class for binary classification metrics. unique_labels : str or array-like (default: 'auto') If 'auto', deduces the unique class labels from y_target Returns score : float Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/scoring/","title":"Scoring"},{"location":"api_modules/mlxtend.evaluate/scoring/#scoring","text":"scoring(y_target, y_predicted, metric='error', positive_label=1, unique_labels='auto') Compute a scoring metric for supervised learning. Parameters y_target : array-like, shape=[n_values] True class labels or target values. y_predicted : array-like, shape=[n_values] Predicted class labels or target values. metric : str (default: 'error') Performance metric: 'accuracy': (TP + TN)/(FP + FN + TP + TN) = 1-ERR 'average per-class accuracy': Average per-class accuracy 'average per-class error': Average per-class error 'balanced per-class accuracy': Average per-class accuracy 'balanced per-class error': Average per-class error 'error': (TP + TN)/(FP+ FN + TP + TN) = 1-ACC 'false_positive_rate': FP/N = FP/(FP + TN) 'true_positive_rate': TP/P = TP/(FN + TP) 'true_negative_rate': TN/N = TN/(FP + TN) 'precision': TP/(TP + FP) 'recall': equal to 'true_positive_rate' 'sensitivity': equal to 'true_positive_rate' or 'recall' 'specificity': equal to 'true_negative_rate' 'f1': 2 * (PRE * REC)/(PRE + REC) 'matthews_corr_coef': (TP TN - FP FN) / (sqrt{(TP + FP)( TP + FN )( TN + FP )( TN + FN )}) Where: [TP: True positives, TN = True negatives, TN: True negatives, FN = False negatives] positive_label : int (default: 1) Label of the positive class for binary classification metrics. unique_labels : str or array-like (default: 'auto') If 'auto', deduces the unique class labels from y_target Returns score : float Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/scoring/","title":"scoring"},{"location":"api_modules/mlxtend.feature_extraction/LinearDiscriminantAnalysis/","text":"LinearDiscriminantAnalysis LinearDiscriminantAnalysis(n_discriminants=None) Linear Discriminant Analysis Class Parameters n_discriminants : int (default: None) The number of discrimants for transformation. Keeps the original dimensions of the dataset if None . Attributes w_ : array-like, shape=[n_features, n_discriminants] Projection matrix e_vals_ : array-like, shape=[n_features] Eigenvalues in sorted order. e_vecs_ : array-like, shape=[n_features] Eigenvectors in sorted order. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/feature_extraction/LinearDiscriminantAnalysis/ Methods fit(X, y, n_classes=None) Fit the LDA model with X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values. n_classes : int (default: None) A positive integer to declare the number of class labels if not all class labels are present in a partial training set. Gets the number of class labels automatically if None. Returns self : object get_params(deep=True) Get parameters for this estimator. Parameters deep : boolean, optional If True, will return the parameters for this estimator and contained subobjects that are estimators. Returns params : mapping of string to any Parameter names mapped to their values.' adapted from https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/base.py Author: Gael Varoquaux gael.varoquaux@normalesup.org License: BSD 3 clause set_params( params) Set the parameters of this estimator. The method works on simple estimators as well as on nested objects (such as pipelines). The latter have parameters of the form <component>__<parameter> so that it's possible to update each component of a nested object. Returns self adapted from https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/base.py Author: Gael Varoquaux <gael.varoquaux@normalesup.org> License: BSD 3 clause transform(X) Apply the linear transformation on X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns X_projected : np.ndarray, shape = [n_samples, n_discriminants] Projected training vectors.","title":"LinearDiscriminantAnalysis"},{"location":"api_modules/mlxtend.feature_extraction/LinearDiscriminantAnalysis/#lineardiscriminantanalysis","text":"LinearDiscriminantAnalysis(n_discriminants=None) Linear Discriminant Analysis Class Parameters n_discriminants : int (default: None) The number of discrimants for transformation. Keeps the original dimensions of the dataset if None . Attributes w_ : array-like, shape=[n_features, n_discriminants] Projection matrix e_vals_ : array-like, shape=[n_features] Eigenvalues in sorted order. e_vecs_ : array-like, shape=[n_features] Eigenvectors in sorted order. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/feature_extraction/LinearDiscriminantAnalysis/","title":"LinearDiscriminantAnalysis"},{"location":"api_modules/mlxtend.feature_extraction/LinearDiscriminantAnalysis/#methods","text":"fit(X, y, n_classes=None) Fit the LDA model with X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values. n_classes : int (default: None) A positive integer to declare the number of class labels if not all class labels are present in a partial training set. Gets the number of class labels automatically if None. Returns self : object get_params(deep=True) Get parameters for this estimator. Parameters deep : boolean, optional If True, will return the parameters for this estimator and contained subobjects that are estimators. Returns params : mapping of string to any Parameter names mapped to their values.' adapted from https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/base.py Author: Gael Varoquaux gael.varoquaux@normalesup.org License: BSD 3 clause set_params( params) Set the parameters of this estimator. The method works on simple estimators as well as on nested objects (such as pipelines). The latter have parameters of the form <component>__<parameter> so that it's possible to update each component of a nested object. Returns self adapted from https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/base.py Author: Gael Varoquaux <gael.varoquaux@normalesup.org> License: BSD 3 clause transform(X) Apply the linear transformation on X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns X_projected : np.ndarray, shape = [n_samples, n_discriminants] Projected training vectors.","title":"Methods"},{"location":"api_modules/mlxtend.feature_extraction/PrincipalComponentAnalysis/","text":"PrincipalComponentAnalysis PrincipalComponentAnalysis(n_components=None, solver='svd', whitening=False) Principal Component Analysis Class Parameters n_components : int (default: None) The number of principal components for transformation. Keeps the original dimensions of the dataset if None . solver : str (default: 'svd') Method for performing the matrix decomposition. {'eigen', 'svd'} whitening : bool (default: False) Performs whitening such that the covariance matrix of the transformed data will be the identity matrix. Attributes w_ : array-like, shape=[n_features, n_components] Projection matrix e_vals_ : array-like, shape=[n_features] Eigenvalues in sorted order. e_vecs_ : array-like, shape=[n_features] Eigenvectors in sorted order. e_vals_normalized_ : array-like, shape=[n_features] Normalized eigen values such that they sum up to 1. This is equal to what's often referred to as \"explained variance ratios.\" loadings_ : array_like, shape=[n_features, n_features] The factor loadings of the original variables onto the principal components. The columns are the principal components, and the rows are the features loadings. For instance, the first column contains the loadings onto the first principal component. Note that the signs may be flipped depending on whether you use the 'eigen' or 'svd' solver; this does not affect the interpretation of the loadings though. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/feature_extraction/PrincipalComponentAnalysis/ Methods fit(X, y=None) Learn model from training data. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns self : object get_params(deep=True) Get parameters for this estimator. Parameters deep : boolean, optional If True, will return the parameters for this estimator and contained subobjects that are estimators. Returns params : mapping of string to any Parameter names mapped to their values.' adapted from https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/base.py Author: Gael Varoquaux gael.varoquaux@normalesup.org License: BSD 3 clause set_params( params) Set the parameters of this estimator. The method works on simple estimators as well as on nested objects (such as pipelines). The latter have parameters of the form <component>__<parameter> so that it's possible to update each component of a nested object. Returns self adapted from https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/base.py Author: Gael Varoquaux <gael.varoquaux@normalesup.org> License: BSD 3 clause transform(X) Apply the linear transformation on X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns X_projected : np.ndarray, shape = [n_samples, n_components] Projected training vectors.","title":"PrincipalComponentAnalysis"},{"location":"api_modules/mlxtend.feature_extraction/PrincipalComponentAnalysis/#principalcomponentanalysis","text":"PrincipalComponentAnalysis(n_components=None, solver='svd', whitening=False) Principal Component Analysis Class Parameters n_components : int (default: None) The number of principal components for transformation. Keeps the original dimensions of the dataset if None . solver : str (default: 'svd') Method for performing the matrix decomposition. {'eigen', 'svd'} whitening : bool (default: False) Performs whitening such that the covariance matrix of the transformed data will be the identity matrix. Attributes w_ : array-like, shape=[n_features, n_components] Projection matrix e_vals_ : array-like, shape=[n_features] Eigenvalues in sorted order. e_vecs_ : array-like, shape=[n_features] Eigenvectors in sorted order. e_vals_normalized_ : array-like, shape=[n_features] Normalized eigen values such that they sum up to 1. This is equal to what's often referred to as \"explained variance ratios.\" loadings_ : array_like, shape=[n_features, n_features] The factor loadings of the original variables onto the principal components. The columns are the principal components, and the rows are the features loadings. For instance, the first column contains the loadings onto the first principal component. Note that the signs may be flipped depending on whether you use the 'eigen' or 'svd' solver; this does not affect the interpretation of the loadings though. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/feature_extraction/PrincipalComponentAnalysis/","title":"PrincipalComponentAnalysis"},{"location":"api_modules/mlxtend.feature_extraction/PrincipalComponentAnalysis/#methods","text":"fit(X, y=None) Learn model from training data. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns self : object get_params(deep=True) Get parameters for this estimator. Parameters deep : boolean, optional If True, will return the parameters for this estimator and contained subobjects that are estimators. Returns params : mapping of string to any Parameter names mapped to their values.' adapted from https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/base.py Author: Gael Varoquaux gael.varoquaux@normalesup.org License: BSD 3 clause set_params( params) Set the parameters of this estimator. The method works on simple estimators as well as on nested objects (such as pipelines). The latter have parameters of the form <component>__<parameter> so that it's possible to update each component of a nested object. Returns self adapted from https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/base.py Author: Gael Varoquaux <gael.varoquaux@normalesup.org> License: BSD 3 clause transform(X) Apply the linear transformation on X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns X_projected : np.ndarray, shape = [n_samples, n_components] Projected training vectors.","title":"Methods"},{"location":"api_modules/mlxtend.feature_extraction/RBFKernelPCA/","text":"RBFKernelPCA RBFKernelPCA(gamma=15.0, n_components=None, copy_X=True) RBF Kernel Principal Component Analysis for dimensionality reduction. Parameters gamma : float (default: 15.0) Free parameter (coefficient) of the RBF kernel. n_components : int (default: None) The number of principal components for transformation. Keeps the original dimensions of the dataset if None . copy_X : bool (default: True) Copies training data, which is required to compute the projection of new data via the transform method. Uses a reference to X if False. Attributes e_vals_ : array-like, shape=[n_features] Eigenvalues in sorted order. e_vecs_ : array-like, shape=[n_features] Eigenvectors in sorted order. X_projected_ : array-like, shape=[n_samples, n_components] Training samples projected along the component axes. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/feature_extraction/RBFKernelPCA/ Methods fit(X) Learn model from training data. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns self : object get_params(deep=True) Get parameters for this estimator. Parameters deep : boolean, optional If True, will return the parameters for this estimator and contained subobjects that are estimators. Returns params : mapping of string to any Parameter names mapped to their values.' adapted from https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/base.py Author: Gael Varoquaux gael.varoquaux@normalesup.org License: BSD 3 clause set_params( params) Set the parameters of this estimator. The method works on simple estimators as well as on nested objects (such as pipelines). The latter have parameters of the form <component>__<parameter> so that it's possible to update each component of a nested object. Returns self adapted from https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/base.py Author: Gael Varoquaux <gael.varoquaux@normalesup.org> License: BSD 3 clause transform(X) Apply the non-linear transformation on X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns X_projected : np.ndarray, shape = [n_samples, n_components] Projected training vectors.","title":"RBFKernelPCA"},{"location":"api_modules/mlxtend.feature_extraction/RBFKernelPCA/#rbfkernelpca","text":"RBFKernelPCA(gamma=15.0, n_components=None, copy_X=True) RBF Kernel Principal Component Analysis for dimensionality reduction. Parameters gamma : float (default: 15.0) Free parameter (coefficient) of the RBF kernel. n_components : int (default: None) The number of principal components for transformation. Keeps the original dimensions of the dataset if None . copy_X : bool (default: True) Copies training data, which is required to compute the projection of new data via the transform method. Uses a reference to X if False. Attributes e_vals_ : array-like, shape=[n_features] Eigenvalues in sorted order. e_vecs_ : array-like, shape=[n_features] Eigenvectors in sorted order. X_projected_ : array-like, shape=[n_samples, n_components] Training samples projected along the component axes. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/feature_extraction/RBFKernelPCA/","title":"RBFKernelPCA"},{"location":"api_modules/mlxtend.feature_extraction/RBFKernelPCA/#methods","text":"fit(X) Learn model from training data. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns self : object get_params(deep=True) Get parameters for this estimator. Parameters deep : boolean, optional If True, will return the parameters for this estimator and contained subobjects that are estimators. Returns params : mapping of string to any Parameter names mapped to their values.' adapted from https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/base.py Author: Gael Varoquaux gael.varoquaux@normalesup.org License: BSD 3 clause set_params( params) Set the parameters of this estimator. The method works on simple estimators as well as on nested objects (such as pipelines). The latter have parameters of the form <component>__<parameter> so that it's possible to update each component of a nested object. Returns self adapted from https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/base.py Author: Gael Varoquaux <gael.varoquaux@normalesup.org> License: BSD 3 clause transform(X) Apply the non-linear transformation on X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns X_projected : np.ndarray, shape = [n_samples, n_components] Projected training vectors.","title":"Methods"},{"location":"api_modules/mlxtend.feature_selection/ColumnSelector/","text":"ColumnSelector ColumnSelector(cols=None, drop_axis=False) Object for selecting specific columns from a data set. Parameters cols : array-like (default: None) A list specifying the feature indices to be selected. For example, [1, 4, 5] to select the 2nd, 5th, and 6th feature columns, and ['A','C','D'] to select the name of feature columns A, C and D. If None, returns all columns in the array. drop_axis : bool (default=False) Drops last axis if True and the only one column is selected. This is useful, e.g., when the ColumnSelector is used for selecting only one column and the resulting array should be fed to e.g., a scikit-learn column selector. E.g., instead of returning an array with shape (n_samples, 1), drop_axis=True will return an aray with shape (n_samples,). Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/feature_selection/ColumnSelector/ Methods fit(X, y=None) Mock method. Does nothing. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] (default: None) Returns self fit_transform(X, y=None) Return a slice of the input array. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] (default: None) Returns X_slice : shape = [n_samples, k_features] Subset of the feature space where k_features <= n_features get_params(deep=True) Get parameters for this estimator. Parameters deep : bool, default=True If True, will return the parameters for this estimator and contained subobjects that are estimators. Returns params : dict Parameter names mapped to their values. set_params( params) Set the parameters of this estimator. The method works on simple estimators as well as on nested objects (such as :class:`~sklearn.pipeline.Pipeline`). The latter have parameters of the form ``<component>__<parameter>`` so that it's possible to update each component of a nested object. Parameters **params : dict Estimator parameters. Returns self : estimator instance Estimator instance. transform(X, y=None) Return a slice of the input array. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] (default: None) Returns X_slice : shape = [n_samples, k_features] Subset of the feature space where k_features <= n_features","title":"ColumnSelector"},{"location":"api_modules/mlxtend.feature_selection/ColumnSelector/#columnselector","text":"ColumnSelector(cols=None, drop_axis=False) Object for selecting specific columns from a data set. Parameters cols : array-like (default: None) A list specifying the feature indices to be selected. For example, [1, 4, 5] to select the 2nd, 5th, and 6th feature columns, and ['A','C','D'] to select the name of feature columns A, C and D. If None, returns all columns in the array. drop_axis : bool (default=False) Drops last axis if True and the only one column is selected. This is useful, e.g., when the ColumnSelector is used for selecting only one column and the resulting array should be fed to e.g., a scikit-learn column selector. E.g., instead of returning an array with shape (n_samples, 1), drop_axis=True will return an aray with shape (n_samples,). Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/feature_selection/ColumnSelector/","title":"ColumnSelector"},{"location":"api_modules/mlxtend.feature_selection/ColumnSelector/#methods","text":"fit(X, y=None) Mock method. Does nothing. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] (default: None) Returns self fit_transform(X, y=None) Return a slice of the input array. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] (default: None) Returns X_slice : shape = [n_samples, k_features] Subset of the feature space where k_features <= n_features get_params(deep=True) Get parameters for this estimator. Parameters deep : bool, default=True If True, will return the parameters for this estimator and contained subobjects that are estimators. Returns params : dict Parameter names mapped to their values. set_params( params) Set the parameters of this estimator. The method works on simple estimators as well as on nested objects (such as :class:`~sklearn.pipeline.Pipeline`). The latter have parameters of the form ``<component>__<parameter>`` so that it's possible to update each component of a nested object. Parameters **params : dict Estimator parameters. Returns self : estimator instance Estimator instance. transform(X, y=None) Return a slice of the input array. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] (default: None) Returns X_slice : shape = [n_samples, k_features] Subset of the feature space where k_features <= n_features","title":"Methods"},{"location":"api_modules/mlxtend.feature_selection/ExhaustiveFeatureSelector/","text":"ExhaustiveFeatureSelector ExhaustiveFeatureSelector(estimator, min_features=1, max_features=1, print_progress=True, scoring='accuracy', cv=5, n_jobs=1, pre_dispatch='2 n_jobs', clone_estimator=True, fixed_features=None, feature_groups=None)* Exhaustive Feature Selection for Classification and Regression. (new in v0.4.3) Parameters estimator : scikit-learn classifier or regressor min_features : int (default: 1) Minumum number of features to select max_features : int (default: 1) Maximum number of features to select. If parameter feature_groups is not None, the number of features is equal to the number of feature groups, i.e. len(feature_groups) . For example, if feature_groups = [[0], [1], [2, 3], [4]] , then the max_features value cannot exceed 4. print_progress : bool (default: True) Prints progress as the number of epochs to stderr. scoring : str, (default='accuracy') Scoring metric in {accuracy, f1, precision, recall, roc_auc} for classifiers, {'mean_absolute_error', 'mean_squared_error', 'median_absolute_error', 'r2'} for regressors, or a callable object or function with signature scorer(estimator, X, y) . cv : int (default: 5) Scikit-learn cross-validation generator or int . If estimator is a classifier (or y consists of integer class labels), stratified k-fold is performed, and regular k-fold cross-validation otherwise. No cross-validation if cv is None, False, or 0. n_jobs : int (default: 1) The number of CPUs to use for evaluating different feature subsets in parallel. -1 means 'all CPUs'. pre_dispatch : int, or string (default: '2*n_jobs') Controls the number of jobs that get dispatched during parallel execution if n_jobs > 1 or n_jobs=-1 . Reducing this number can be useful to avoid an explosion of memory consumption when more jobs get dispatched than CPUs can process. This parameter can be: None, in which case all the jobs are immediately created and spawned. Use this for lightweight and fast-running jobs, to avoid delays due to on-demand spawning of the jobs An int, giving the exact number of total jobs that are spawned A string, giving an expression as a function of n_jobs, as in 2*n_jobs clone_estimator : bool (default: True) Clones estimator if True; works with the original estimator instance if False. Set to False if the estimator doesn't implement scikit-learn's set_params and get_params methods. In addition, it is required to set cv=0, and n_jobs=1. fixed_features : tuple (default: None) If not None , the feature indices provided as a tuple will be regarded as fixed by the feature selector. For example, if fixed_features=(1, 3, 7) , the 2nd, 4th, and 8th feature are guaranteed to be present in the solution. Note that if fixed_features is not None , make sure that the number of features to be selected is greater than len(fixed_features) . In other words, ensure that k_features > len(fixed_features) . feature_groups : list or None (default: None) Optional argument for treating certain features as a group. This means, the features within a group are always selected together, never split. For example, feature_groups=[[1], [2], [3, 4, 5]] specifies 3 feature groups.In this case, possible feature selection results with k_features=2 are [[1], [2] , [[1], [3, 4, 5]] , or [[2], [3, 4, 5]] . Feature groups can be useful for interpretability, for example, if features 3, 4, 5 are one-hot encoded features. (For more details, please read the notes at the bottom of this docstring). New in mlxtend v. 0.21.0. Attributes best_idx_ : array-like, shape = [n_predictions] Feature Indices of the selected feature subsets. best_feature_names_ : array-like, shape = [n_predictions] Feature names of the selected feature subsets. If pandas DataFrames are used in the fit method, the feature names correspond to the column names. Otherwise, the feature names are string representation of the feature array indices. New in v 0.13.0. best_score_ : float Cross validation average score of the selected subset. subsets_ : dict A dictionary of selected feature subsets during the exhaustive selection, where the dictionary keys are the lengths k of these feature subsets. The dictionary values are dictionaries themselves with the following keys: 'feature_idx' (tuple of indices of the feature subset) 'feature_names' (tuple of feature names of the feat. subset) 'cv_scores' (list individual cross-validation scores) 'avg_score' (average cross-validation score) Note that if pandas DataFrames are used in the fit method, the 'feature_names' correspond to the column names. Otherwise, the feature names are string representation of the feature array indices. The 'feature_names' is new in v. 0.13.0. Notes (1) If parameter feature_groups is not None, the number of features is equal to the number of feature groups, i.e. len(feature_groups) . For example, if feature_groups = [[0], [1], [2, 3], [4]] , then the max_features value cannot exceed 4. (2) Although two or more individual features may be considered as one group throughout the feature-selection process, it does not mean the individual features of that group have the same impact on the outcome. For instance, in linear regression, the coefficient of the feature 2 and 3 can be different even if they are considered as one group in feature_groups. (3) If both fixed_features and feature_groups are specified, ensure that each feature group contains the fixed_features selection. E.g., for a 3-feature set fixed_features=[0, 1] and feature_groups=[[0, 1], [2]] is valid; fixed_features=[0, 1] and feature_groups=[[0], [1, 2]] is not valid. (4) In case of KeyboardInterrupt, the dictionary subsets may not be completed. If user is still interested in getting the best score, they can use method `finalize_fit`. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/feature_selection/ExhaustiveFeatureSelector/ Methods finalize_fit() None fit(X, y, groups=None, fit_params) Perform feature selection and learn model from training data. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. New in v 0.13.0: pandas DataFrames are now also accepted as argument for X. y : array-like, shape = [n_samples] Target values. groups : array-like, with shape (n_samples,), optional Group labels for the samples used while splitting the dataset into train/test set. Passed to the fit method of the cross-validator. fit_params : dict of string -> object, optional Parameters to pass to to the fit method of classifier. Returns self : object fit_transform(X, y, groups=None, fit_params) Fit to training data and return the best selected features from X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. New in v 0.13.0: pandas DataFrames are now also accepted as argument for X. y : array-like, shape = [n_samples] Target values. groups : array-like, with shape (n_samples,), optional Group labels for the samples used while splitting the dataset into train/test set. Passed to the fit method of the cross-validator. fit_params : dict of string -> object, optional Parameters to pass to to the fit method of classifier. Returns Feature subset of X, shape={n_samples, k_features} get_metric_dict(confidence_interval=0.95) Return metric dictionary Parameters confidence_interval : float (default: 0.95) A positive float between 0.0 and 1.0 to compute the confidence interval bounds of the CV score averages. Returns Dictionary with items where each dictionary value is a list with the number of iterations (number of feature subsets) as its length. The dictionary keys corresponding to these lists are as follows: 'feature_idx': tuple of the indices of the feature subset 'cv_scores': list with individual CV scores 'avg_score': of CV average scores 'std_dev': standard deviation of the CV score average 'std_err': standard error of the CV score average 'ci_bound': confidence interval bound of the CV score average get_params(deep=True) Get parameters for this estimator. Parameters deep : bool, default=True If True, will return the parameters for this estimator and contained subobjects that are estimators. Returns params : dict Parameter names mapped to their values. set_params( params) Set the parameters of this estimator. The method works on simple estimators as well as on nested objects (such as :class:`~sklearn.pipeline.Pipeline`). The latter have parameters of the form ``<component>__<parameter>`` so that it's possible to update each component of a nested object. Parameters **params : dict Estimator parameters. Returns self : estimator instance Estimator instance. transform(X) Return the best selected features from X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. New in v 0.13.0: pandas DataFrames are now also accepted as argument for X. Returns Feature subset of X, shape={n_samples, k_features}","title":"ExhaustiveFeatureSelector"},{"location":"api_modules/mlxtend.feature_selection/ExhaustiveFeatureSelector/#exhaustivefeatureselector","text":"ExhaustiveFeatureSelector(estimator, min_features=1, max_features=1, print_progress=True, scoring='accuracy', cv=5, n_jobs=1, pre_dispatch='2 n_jobs', clone_estimator=True, fixed_features=None, feature_groups=None)* Exhaustive Feature Selection for Classification and Regression. (new in v0.4.3) Parameters estimator : scikit-learn classifier or regressor min_features : int (default: 1) Minumum number of features to select max_features : int (default: 1) Maximum number of features to select. If parameter feature_groups is not None, the number of features is equal to the number of feature groups, i.e. len(feature_groups) . For example, if feature_groups = [[0], [1], [2, 3], [4]] , then the max_features value cannot exceed 4. print_progress : bool (default: True) Prints progress as the number of epochs to stderr. scoring : str, (default='accuracy') Scoring metric in {accuracy, f1, precision, recall, roc_auc} for classifiers, {'mean_absolute_error', 'mean_squared_error', 'median_absolute_error', 'r2'} for regressors, or a callable object or function with signature scorer(estimator, X, y) . cv : int (default: 5) Scikit-learn cross-validation generator or int . If estimator is a classifier (or y consists of integer class labels), stratified k-fold is performed, and regular k-fold cross-validation otherwise. No cross-validation if cv is None, False, or 0. n_jobs : int (default: 1) The number of CPUs to use for evaluating different feature subsets in parallel. -1 means 'all CPUs'. pre_dispatch : int, or string (default: '2*n_jobs') Controls the number of jobs that get dispatched during parallel execution if n_jobs > 1 or n_jobs=-1 . Reducing this number can be useful to avoid an explosion of memory consumption when more jobs get dispatched than CPUs can process. This parameter can be: None, in which case all the jobs are immediately created and spawned. Use this for lightweight and fast-running jobs, to avoid delays due to on-demand spawning of the jobs An int, giving the exact number of total jobs that are spawned A string, giving an expression as a function of n_jobs, as in 2*n_jobs clone_estimator : bool (default: True) Clones estimator if True; works with the original estimator instance if False. Set to False if the estimator doesn't implement scikit-learn's set_params and get_params methods. In addition, it is required to set cv=0, and n_jobs=1. fixed_features : tuple (default: None) If not None , the feature indices provided as a tuple will be regarded as fixed by the feature selector. For example, if fixed_features=(1, 3, 7) , the 2nd, 4th, and 8th feature are guaranteed to be present in the solution. Note that if fixed_features is not None , make sure that the number of features to be selected is greater than len(fixed_features) . In other words, ensure that k_features > len(fixed_features) . feature_groups : list or None (default: None) Optional argument for treating certain features as a group. This means, the features within a group are always selected together, never split. For example, feature_groups=[[1], [2], [3, 4, 5]] specifies 3 feature groups.In this case, possible feature selection results with k_features=2 are [[1], [2] , [[1], [3, 4, 5]] , or [[2], [3, 4, 5]] . Feature groups can be useful for interpretability, for example, if features 3, 4, 5 are one-hot encoded features. (For more details, please read the notes at the bottom of this docstring). New in mlxtend v. 0.21.0. Attributes best_idx_ : array-like, shape = [n_predictions] Feature Indices of the selected feature subsets. best_feature_names_ : array-like, shape = [n_predictions] Feature names of the selected feature subsets. If pandas DataFrames are used in the fit method, the feature names correspond to the column names. Otherwise, the feature names are string representation of the feature array indices. New in v 0.13.0. best_score_ : float Cross validation average score of the selected subset. subsets_ : dict A dictionary of selected feature subsets during the exhaustive selection, where the dictionary keys are the lengths k of these feature subsets. The dictionary values are dictionaries themselves with the following keys: 'feature_idx' (tuple of indices of the feature subset) 'feature_names' (tuple of feature names of the feat. subset) 'cv_scores' (list individual cross-validation scores) 'avg_score' (average cross-validation score) Note that if pandas DataFrames are used in the fit method, the 'feature_names' correspond to the column names. Otherwise, the feature names are string representation of the feature array indices. The 'feature_names' is new in v. 0.13.0. Notes (1) If parameter feature_groups is not None, the number of features is equal to the number of feature groups, i.e. len(feature_groups) . For example, if feature_groups = [[0], [1], [2, 3], [4]] , then the max_features value cannot exceed 4. (2) Although two or more individual features may be considered as one group throughout the feature-selection process, it does not mean the individual features of that group have the same impact on the outcome. For instance, in linear regression, the coefficient of the feature 2 and 3 can be different even if they are considered as one group in feature_groups. (3) If both fixed_features and feature_groups are specified, ensure that each feature group contains the fixed_features selection. E.g., for a 3-feature set fixed_features=[0, 1] and feature_groups=[[0, 1], [2]] is valid; fixed_features=[0, 1] and feature_groups=[[0], [1, 2]] is not valid. (4) In case of KeyboardInterrupt, the dictionary subsets may not be completed. If user is still interested in getting the best score, they can use method `finalize_fit`. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/feature_selection/ExhaustiveFeatureSelector/","title":"ExhaustiveFeatureSelector"},{"location":"api_modules/mlxtend.feature_selection/ExhaustiveFeatureSelector/#methods","text":"finalize_fit() None fit(X, y, groups=None, fit_params) Perform feature selection and learn model from training data. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. New in v 0.13.0: pandas DataFrames are now also accepted as argument for X. y : array-like, shape = [n_samples] Target values. groups : array-like, with shape (n_samples,), optional Group labels for the samples used while splitting the dataset into train/test set. Passed to the fit method of the cross-validator. fit_params : dict of string -> object, optional Parameters to pass to to the fit method of classifier. Returns self : object fit_transform(X, y, groups=None, fit_params) Fit to training data and return the best selected features from X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. New in v 0.13.0: pandas DataFrames are now also accepted as argument for X. y : array-like, shape = [n_samples] Target values. groups : array-like, with shape (n_samples,), optional Group labels for the samples used while splitting the dataset into train/test set. Passed to the fit method of the cross-validator. fit_params : dict of string -> object, optional Parameters to pass to to the fit method of classifier. Returns Feature subset of X, shape={n_samples, k_features} get_metric_dict(confidence_interval=0.95) Return metric dictionary Parameters confidence_interval : float (default: 0.95) A positive float between 0.0 and 1.0 to compute the confidence interval bounds of the CV score averages. Returns Dictionary with items where each dictionary value is a list with the number of iterations (number of feature subsets) as its length. The dictionary keys corresponding to these lists are as follows: 'feature_idx': tuple of the indices of the feature subset 'cv_scores': list with individual CV scores 'avg_score': of CV average scores 'std_dev': standard deviation of the CV score average 'std_err': standard error of the CV score average 'ci_bound': confidence interval bound of the CV score average get_params(deep=True) Get parameters for this estimator. Parameters deep : bool, default=True If True, will return the parameters for this estimator and contained subobjects that are estimators. Returns params : dict Parameter names mapped to their values. set_params( params) Set the parameters of this estimator. The method works on simple estimators as well as on nested objects (such as :class:`~sklearn.pipeline.Pipeline`). The latter have parameters of the form ``<component>__<parameter>`` so that it's possible to update each component of a nested object. Parameters **params : dict Estimator parameters. Returns self : estimator instance Estimator instance. transform(X) Return the best selected features from X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. New in v 0.13.0: pandas DataFrames are now also accepted as argument for X. Returns Feature subset of X, shape={n_samples, k_features}","title":"Methods"},{"location":"api_modules/mlxtend.feature_selection/SequentialFeatureSelector/","text":"SequentialFeatureSelector SequentialFeatureSelector(estimator, k_features=1, forward=True, floating=False, verbose=0, scoring=None, cv=5, n_jobs=1, pre_dispatch='2 n_jobs', clone_estimator=True, fixed_features=None, feature_groups=None)* Sequential Feature Selection for Classification and Regression. Parameters estimator : scikit-learn classifier or regressor k_features : int or tuple or str (default: 1) Number of features to select, where k_features < the full feature set. New in 0.4.2: A tuple containing a min and max value can be provided, and the SFS will consider return any feature combination between min and max that scored highest in cross-validation. For example, the tuple (1, 4) will return any combination from 1 up to 4 features instead of a fixed number of features k. New in 0.8.0: A string argument \"best\" or \"parsimonious\". If \"best\" is provided, the feature selector will return the feature subset with the best cross-validation performance. If \"parsimonious\" is provided as an argument, the smallest feature subset that is within one standard error of the cross-validation performance will be selected. forward : bool (default: True) Forward selection if True, backward selection otherwise floating : bool (default: False) Adds a conditional exclusion/inclusion if True. verbose : int (default: 0), level of verbosity to use in logging. If 0, no output, if 1 number of features in current set, if 2 detailed logging i ncluding timestamp and cv scores at step. scoring : str, callable, or None (default: None) If None (default), uses 'accuracy' for sklearn classifiers and 'r2' for sklearn regressors. If str, uses a sklearn scoring metric string identifier, for example {accuracy, f1, precision, recall, roc_auc} for classifiers, {'mean_absolute_error', 'mean_squared_error'/'neg_mean_squared_error', 'median_absolute_error', 'r2'} for regressors. If a callable object or function is provided, it has to be conform with sklearn's signature scorer(estimator, X, y) ; see https://scikit-learn.org/stable/modules/generated/sklearn.metrics.make_scorer.html for more information. cv : int (default: 5) Integer or iterable yielding train, test splits. If cv is an integer and estimator is a classifier (or y consists of integer class labels) stratified k-fold. Otherwise regular k-fold cross-validation is performed. No cross-validation if cv is None, False, or 0. n_jobs : int (default: 1) The number of CPUs to use for evaluating different feature subsets in parallel. -1 means 'all CPUs'. pre_dispatch : int, or string (default: '2*n_jobs') Controls the number of jobs that get dispatched during parallel execution if n_jobs > 1 or n_jobs=-1 . Reducing this number can be useful to avoid an explosion of memory consumption when more jobs get dispatched than CPUs can process. This parameter can be: None, in which case all the jobs are immediately created and spawned. Use this for lightweight and fast-running jobs, to avoid delays due to on-demand spawning of the jobs An int, giving the exact number of total jobs that are spawned A string, giving an expression as a function of n_jobs, as in 2*n_jobs clone_estimator : bool (default: True) Clones estimator if True; works with the original estimator instance if False. Set to False if the estimator doesn't implement scikit-learn's set_params and get_params methods. In addition, it is required to set cv=0, and n_jobs=1. fixed_features : tuple (default: None) If not None , the feature indices provided as a tuple will be regarded as fixed by the feature selector. For example, if fixed_features=(1, 3, 7) , the 2nd, 4th, and 8th feature are guaranteed to be present in the solution. Note that if fixed_features is not None , make sure that the number of features to be selected is greater than len(fixed_features) . In other words, ensure that k_features > len(fixed_features) . New in mlxtend v. 0.18.0. feature_groups : list or None (default: None) Optional argument for treating certain features as a group. This means, the features within a group are always selected together, never split. For example, feature_groups=[[1], [2], [3, 4, 5]] specifies 3 feature groups. In this case, possible feature selection results with k_features=2 are [[1], [2] , [[1], [3, 4, 5]] , or [[2], [3, 4, 5]] . Feature groups can be useful for interpretability, for example, if features 3, 4, 5 are one-hot encoded features. (For more details, please read the notes at the bottom of this docstring). New in mlxtend v. 0.21.0. Attributes k_feature_idx_ : array-like, shape = [n_predictions] Feature Indices of the selected feature subsets. k_feature_names_ : array-like, shape = [n_predictions] Feature names of the selected feature subsets. If pandas DataFrames are used in the fit method, the feature names correspond to the column names. Otherwise, the feature names are string representation of the feature array indices. New in v 0.13.0. k_score_ : float Cross validation average score of the selected subset. subsets_ : dict A dictionary of selected feature subsets during the sequential selection, where the dictionary keys are the lengths k of these feature subsets. If the parameter feature_groups is not None, the value of key indicates the number of groups that are selected together. The dictionary values are dictionaries themselves with the following keys: 'feature_idx' (tuple of indices of the feature subset) 'feature_names' (tuple of feature names of the feat. subset) 'cv_scores' (list individual cross-validation scores) 'avg_score' (average cross-validation score) Note that if pandas DataFrames are used in the fit method, the 'feature_names' correspond to the column names. Otherwise, the feature names are string representation of the feature array indices. The 'feature_names' is new in v 0.13.0. Notes (1) If parameter feature_groups is not None, the number of features is equal to the number of feature groups, i.e. len(feature_groups) . For example, if feature_groups = [[0], [1], [2, 3], [4]] , then the max_features value cannot exceed 4. (2) Although two or more individual features may be considered as one group throughout the feature-selection process, it does not mean the individual features of that group have the same impact on the outcome. For instance, in linear regression, the coefficient of the feature 2 and 3 can be different even if they are considered as one group in feature_groups. (3) If both fixed_features and feature_groups are specified, ensure that each feature group contains the fixed_features selection. E.g., for a 3-feature set fixed_features=[0, 1] and feature_groups=[[0, 1], [2]] is valid; fixed_features=[0, 1] and feature_groups=[[0], [1, 2]] is not valid. (4) In case of KeyboardInterrupt, the dictionary subsets may not be completed. If user is still interested in getting the best score, they can use method `finalize_fit`. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/feature_selection/SequentialFeatureSelector/ Methods finalize_fit() None fit(X, y, groups=None, fit_params) Perform feature selection and learn model from training data. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. New in v 0.13.0: pandas DataFrames are now also accepted as argument for X. y : array-like, shape = [n_samples] Target values. New in v 0.13.0: pandas DataFrames are now also accepted as argument for y. groups : array-like, with shape (n_samples,), optional Group labels for the samples used while splitting the dataset into train/test set. Passed to the fit method of the cross-validator. fit_params : various, optional Additional parameters that are being passed to the estimator. For example, sample_weights=weights . Returns self : object fit_transform(X, y, groups=None, fit_params) Fit to training data then reduce X to its most important features. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. New in v 0.13.0: pandas DataFrames are now also accepted as argument for X. y : array-like, shape = [n_samples] Target values. New in v 0.13.0: a pandas Series are now also accepted as argument for y. groups : array-like, with shape (n_samples,), optional Group labels for the samples used while splitting the dataset into train/test set. Passed to the fit method of the cross-validator. fit_params : various, optional Additional parameters that are being passed to the estimator. For example, sample_weights=weights . Returns Reduced feature subset of X, shape={n_samples, k_features} generate_error_message_k_features(name) None get_metric_dict(confidence_interval=0.95) Return metric dictionary Parameters confidence_interval : float (default: 0.95) A positive float between 0.0 and 1.0 to compute the confidence interval bounds of the CV score averages. Returns Dictionary with items where each dictionary value is a list with the number of iterations (number of feature subsets) as its length. The dictionary keys corresponding to these lists are as follows: 'feature_idx': tuple of the indices of the feature subset 'cv_scores': list with individual CV scores 'avg_score': of CV average scores 'std_dev': standard deviation of the CV score average 'std_err': standard error of the CV score average 'ci_bound': confidence interval bound of the CV score average get_params(deep=True) Get parameters for this estimator. Parameters deep : bool, default=True If True, will return the parameters for this estimator and contained subobjects that are estimators. Returns params : dict Parameter names mapped to their values. set_params( params) Set the parameters of this estimator. Valid parameter keys can be listed with get_params() . Returns self transform(X) Reduce X to its most important features. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. New in v 0.13.0: pandas DataFrames are now also accepted as argument for X. Returns Reduced feature subset of X, shape={n_samples, k_features} Properties named_estimators Returns List of named estimator tuples, like [('svc', SVC(...))]","title":"SequentialFeatureSelector"},{"location":"api_modules/mlxtend.feature_selection/SequentialFeatureSelector/#sequentialfeatureselector","text":"SequentialFeatureSelector(estimator, k_features=1, forward=True, floating=False, verbose=0, scoring=None, cv=5, n_jobs=1, pre_dispatch='2 n_jobs', clone_estimator=True, fixed_features=None, feature_groups=None)* Sequential Feature Selection for Classification and Regression. Parameters estimator : scikit-learn classifier or regressor k_features : int or tuple or str (default: 1) Number of features to select, where k_features < the full feature set. New in 0.4.2: A tuple containing a min and max value can be provided, and the SFS will consider return any feature combination between min and max that scored highest in cross-validation. For example, the tuple (1, 4) will return any combination from 1 up to 4 features instead of a fixed number of features k. New in 0.8.0: A string argument \"best\" or \"parsimonious\". If \"best\" is provided, the feature selector will return the feature subset with the best cross-validation performance. If \"parsimonious\" is provided as an argument, the smallest feature subset that is within one standard error of the cross-validation performance will be selected. forward : bool (default: True) Forward selection if True, backward selection otherwise floating : bool (default: False) Adds a conditional exclusion/inclusion if True. verbose : int (default: 0), level of verbosity to use in logging. If 0, no output, if 1 number of features in current set, if 2 detailed logging i ncluding timestamp and cv scores at step. scoring : str, callable, or None (default: None) If None (default), uses 'accuracy' for sklearn classifiers and 'r2' for sklearn regressors. If str, uses a sklearn scoring metric string identifier, for example {accuracy, f1, precision, recall, roc_auc} for classifiers, {'mean_absolute_error', 'mean_squared_error'/'neg_mean_squared_error', 'median_absolute_error', 'r2'} for regressors. If a callable object or function is provided, it has to be conform with sklearn's signature scorer(estimator, X, y) ; see https://scikit-learn.org/stable/modules/generated/sklearn.metrics.make_scorer.html for more information. cv : int (default: 5) Integer or iterable yielding train, test splits. If cv is an integer and estimator is a classifier (or y consists of integer class labels) stratified k-fold. Otherwise regular k-fold cross-validation is performed. No cross-validation if cv is None, False, or 0. n_jobs : int (default: 1) The number of CPUs to use for evaluating different feature subsets in parallel. -1 means 'all CPUs'. pre_dispatch : int, or string (default: '2*n_jobs') Controls the number of jobs that get dispatched during parallel execution if n_jobs > 1 or n_jobs=-1 . Reducing this number can be useful to avoid an explosion of memory consumption when more jobs get dispatched than CPUs can process. This parameter can be: None, in which case all the jobs are immediately created and spawned. Use this for lightweight and fast-running jobs, to avoid delays due to on-demand spawning of the jobs An int, giving the exact number of total jobs that are spawned A string, giving an expression as a function of n_jobs, as in 2*n_jobs clone_estimator : bool (default: True) Clones estimator if True; works with the original estimator instance if False. Set to False if the estimator doesn't implement scikit-learn's set_params and get_params methods. In addition, it is required to set cv=0, and n_jobs=1. fixed_features : tuple (default: None) If not None , the feature indices provided as a tuple will be regarded as fixed by the feature selector. For example, if fixed_features=(1, 3, 7) , the 2nd, 4th, and 8th feature are guaranteed to be present in the solution. Note that if fixed_features is not None , make sure that the number of features to be selected is greater than len(fixed_features) . In other words, ensure that k_features > len(fixed_features) . New in mlxtend v. 0.18.0. feature_groups : list or None (default: None) Optional argument for treating certain features as a group. This means, the features within a group are always selected together, never split. For example, feature_groups=[[1], [2], [3, 4, 5]] specifies 3 feature groups. In this case, possible feature selection results with k_features=2 are [[1], [2] , [[1], [3, 4, 5]] , or [[2], [3, 4, 5]] . Feature groups can be useful for interpretability, for example, if features 3, 4, 5 are one-hot encoded features. (For more details, please read the notes at the bottom of this docstring). New in mlxtend v. 0.21.0. Attributes k_feature_idx_ : array-like, shape = [n_predictions] Feature Indices of the selected feature subsets. k_feature_names_ : array-like, shape = [n_predictions] Feature names of the selected feature subsets. If pandas DataFrames are used in the fit method, the feature names correspond to the column names. Otherwise, the feature names are string representation of the feature array indices. New in v 0.13.0. k_score_ : float Cross validation average score of the selected subset. subsets_ : dict A dictionary of selected feature subsets during the sequential selection, where the dictionary keys are the lengths k of these feature subsets. If the parameter feature_groups is not None, the value of key indicates the number of groups that are selected together. The dictionary values are dictionaries themselves with the following keys: 'feature_idx' (tuple of indices of the feature subset) 'feature_names' (tuple of feature names of the feat. subset) 'cv_scores' (list individual cross-validation scores) 'avg_score' (average cross-validation score) Note that if pandas DataFrames are used in the fit method, the 'feature_names' correspond to the column names. Otherwise, the feature names are string representation of the feature array indices. The 'feature_names' is new in v 0.13.0. Notes (1) If parameter feature_groups is not None, the number of features is equal to the number of feature groups, i.e. len(feature_groups) . For example, if feature_groups = [[0], [1], [2, 3], [4]] , then the max_features value cannot exceed 4. (2) Although two or more individual features may be considered as one group throughout the feature-selection process, it does not mean the individual features of that group have the same impact on the outcome. For instance, in linear regression, the coefficient of the feature 2 and 3 can be different even if they are considered as one group in feature_groups. (3) If both fixed_features and feature_groups are specified, ensure that each feature group contains the fixed_features selection. E.g., for a 3-feature set fixed_features=[0, 1] and feature_groups=[[0, 1], [2]] is valid; fixed_features=[0, 1] and feature_groups=[[0], [1, 2]] is not valid. (4) In case of KeyboardInterrupt, the dictionary subsets may not be completed. If user is still interested in getting the best score, they can use method `finalize_fit`. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/feature_selection/SequentialFeatureSelector/","title":"SequentialFeatureSelector"},{"location":"api_modules/mlxtend.feature_selection/SequentialFeatureSelector/#methods","text":"finalize_fit() None fit(X, y, groups=None, fit_params) Perform feature selection and learn model from training data. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. New in v 0.13.0: pandas DataFrames are now also accepted as argument for X. y : array-like, shape = [n_samples] Target values. New in v 0.13.0: pandas DataFrames are now also accepted as argument for y. groups : array-like, with shape (n_samples,), optional Group labels for the samples used while splitting the dataset into train/test set. Passed to the fit method of the cross-validator. fit_params : various, optional Additional parameters that are being passed to the estimator. For example, sample_weights=weights . Returns self : object fit_transform(X, y, groups=None, fit_params) Fit to training data then reduce X to its most important features. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. New in v 0.13.0: pandas DataFrames are now also accepted as argument for X. y : array-like, shape = [n_samples] Target values. New in v 0.13.0: a pandas Series are now also accepted as argument for y. groups : array-like, with shape (n_samples,), optional Group labels for the samples used while splitting the dataset into train/test set. Passed to the fit method of the cross-validator. fit_params : various, optional Additional parameters that are being passed to the estimator. For example, sample_weights=weights . Returns Reduced feature subset of X, shape={n_samples, k_features} generate_error_message_k_features(name) None get_metric_dict(confidence_interval=0.95) Return metric dictionary Parameters confidence_interval : float (default: 0.95) A positive float between 0.0 and 1.0 to compute the confidence interval bounds of the CV score averages. Returns Dictionary with items where each dictionary value is a list with the number of iterations (number of feature subsets) as its length. The dictionary keys corresponding to these lists are as follows: 'feature_idx': tuple of the indices of the feature subset 'cv_scores': list with individual CV scores 'avg_score': of CV average scores 'std_dev': standard deviation of the CV score average 'std_err': standard error of the CV score average 'ci_bound': confidence interval bound of the CV score average get_params(deep=True) Get parameters for this estimator. Parameters deep : bool, default=True If True, will return the parameters for this estimator and contained subobjects that are estimators. Returns params : dict Parameter names mapped to their values. set_params( params) Set the parameters of this estimator. Valid parameter keys can be listed with get_params() . Returns self transform(X) Reduce X to its most important features. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. New in v 0.13.0: pandas DataFrames are now also accepted as argument for X. Returns Reduced feature subset of X, shape={n_samples, k_features}","title":"Methods"},{"location":"api_modules/mlxtend.feature_selection/SequentialFeatureSelector/#properties","text":"named_estimators Returns List of named estimator tuples, like [('svc', SVC(...))]","title":"Properties"},{"location":"api_modules/mlxtend.file_io/find_filegroups/","text":"find_filegroups find_filegroups(paths, substring='', extensions=None, validity_check=True, ignore_invisible=True, rstrip='', ignore_substring=None) Find and collect files from different directories in a python dictionary. Parameters paths : list Paths of the directories to be searched. Dictionary keys are build from the first directory. substring : str (default: '') Substring that all files have to contain to be considered. extensions : list (default: None) None or list of allowed file extensions for each path. If provided, the number of extensions must match the number of paths . validity_check : bool (default: None) If True , checks if all dictionary values have the same number of file paths. Prints a warning and returns an empty dictionary if the validity check failed. ignore_invisible : bool (default: True) If True , ignores invisible files (i.e., files starting with a period). rstrip : str (default: '') If provided, strips characters from right side of the file base names after splitting the extension. Useful to trim different filenames to a common stem. E.g,. \"abc_d.txt\" and \"abc_d_.csv\" would share the stem \"abc_d\" if rstrip is set to \"_\". ignore_substring : str (default: None) Ignores files that contain the specified substring. Returns groups : dict Dictionary of files paths. Keys are the file names found in the first directory listed in paths (without file extension). Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/file_io/find_filegroups/","title":"Find filegroups"},{"location":"api_modules/mlxtend.file_io/find_filegroups/#find_filegroups","text":"find_filegroups(paths, substring='', extensions=None, validity_check=True, ignore_invisible=True, rstrip='', ignore_substring=None) Find and collect files from different directories in a python dictionary. Parameters paths : list Paths of the directories to be searched. Dictionary keys are build from the first directory. substring : str (default: '') Substring that all files have to contain to be considered. extensions : list (default: None) None or list of allowed file extensions for each path. If provided, the number of extensions must match the number of paths . validity_check : bool (default: None) If True , checks if all dictionary values have the same number of file paths. Prints a warning and returns an empty dictionary if the validity check failed. ignore_invisible : bool (default: True) If True , ignores invisible files (i.e., files starting with a period). rstrip : str (default: '') If provided, strips characters from right side of the file base names after splitting the extension. Useful to trim different filenames to a common stem. E.g,. \"abc_d.txt\" and \"abc_d_.csv\" would share the stem \"abc_d\" if rstrip is set to \"_\". ignore_substring : str (default: None) Ignores files that contain the specified substring. Returns groups : dict Dictionary of files paths. Keys are the file names found in the first directory listed in paths (without file extension). Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/file_io/find_filegroups/","title":"find_filegroups"},{"location":"api_modules/mlxtend.file_io/find_files/","text":"find_files find_files(substring, path, recursive=False, check_ext=None, ignore_invisible=True, ignore_substring=None) Find files in a directory based on substring matching. Parameters substring : str Substring of the file to be matched. path : str Path where to look. recursive : bool If true, searches subdirectories recursively. check_ext : str If string (e.g., '.txt'), only returns files that match the specified file extension. ignore_invisible : bool If True , ignores invisible files (i.e., files starting with a period). ignore_substring : str Ignores files that contain the specified substring. Returns results : list List of the matched files. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/file_io/find_files/","title":"Find files"},{"location":"api_modules/mlxtend.file_io/find_files/#find_files","text":"find_files(substring, path, recursive=False, check_ext=None, ignore_invisible=True, ignore_substring=None) Find files in a directory based on substring matching. Parameters substring : str Substring of the file to be matched. path : str Path where to look. recursive : bool If true, searches subdirectories recursively. check_ext : str If string (e.g., '.txt'), only returns files that match the specified file extension. ignore_invisible : bool If True , ignores invisible files (i.e., files starting with a period). ignore_substring : str Ignores files that contain the specified substring. Returns results : list List of the matched files. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/file_io/find_files/","title":"find_files"},{"location":"api_modules/mlxtend.frequent_patterns/apriori/","text":"apriori apriori(df, min_support=0.5, use_colnames=False, max_len=None, verbose=0, low_memory=False) Get frequent itemsets from a one-hot DataFrame Parameters df : pandas DataFrame pandas DataFrame the encoded format. Also supports DataFrames with sparse data; for more info, please see (https://pandas.pydata.org/pandas-docs/stable/ user_guide/sparse.html#sparse-data-structures) Please note that the old pandas SparseDataFrame format is no longer supported in mlxtend >= 0.17.2. The allowed values are either 0/1 or True/False. For example, Apple Bananas Beer Chicken Milk Rice 0 True False True True False True 1 True False True False False True 2 True False True False False False 3 True True False False False False 4 False False True True True True 5 False False True False True True 6 False False True False True False 7 True True False False False False min_support : float (default: 0.5) A float between 0 and 1 for minumum support of the itemsets returned. The support is computed as the fraction transactions_where_item(s)_occur / total_transactions . use_colnames : bool (default: False) If True , uses the DataFrames' column names in the returned DataFrame instead of column indices. max_len : int (default: None) Maximum length of the itemsets generated. If None (default) all possible itemsets lengths (under the apriori condition) are evaluated. verbose : int (default: 0) Shows the number of iterations if >= 1 and low_memory is True . If =1 and low_memory is False , shows the number of combinations. low_memory : bool (default: False) If True , uses an iterator to search for combinations above min_support . Note that while low_memory=True should only be used for large dataset if memory resources are limited, because this implementation is approx. 3-6x slower than the default. Returns pandas DataFrame with columns ['support', 'itemsets'] of all itemsets that are >= min_support and < than max_len (if max_len is not None). Each itemset in the 'itemsets' column is of type frozenset , which is a Python built-in type that behaves similarly to sets except that it is immutable (For more info, see https://docs.python.org/3.6/library/stdtypes.html#frozenset). Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/frequent_patterns/apriori/","title":"Apriori"},{"location":"api_modules/mlxtend.frequent_patterns/apriori/#apriori","text":"apriori(df, min_support=0.5, use_colnames=False, max_len=None, verbose=0, low_memory=False) Get frequent itemsets from a one-hot DataFrame Parameters df : pandas DataFrame pandas DataFrame the encoded format. Also supports DataFrames with sparse data; for more info, please see (https://pandas.pydata.org/pandas-docs/stable/ user_guide/sparse.html#sparse-data-structures) Please note that the old pandas SparseDataFrame format is no longer supported in mlxtend >= 0.17.2. The allowed values are either 0/1 or True/False. For example, Apple Bananas Beer Chicken Milk Rice 0 True False True True False True 1 True False True False False True 2 True False True False False False 3 True True False False False False 4 False False True True True True 5 False False True False True True 6 False False True False True False 7 True True False False False False min_support : float (default: 0.5) A float between 0 and 1 for minumum support of the itemsets returned. The support is computed as the fraction transactions_where_item(s)_occur / total_transactions . use_colnames : bool (default: False) If True , uses the DataFrames' column names in the returned DataFrame instead of column indices. max_len : int (default: None) Maximum length of the itemsets generated. If None (default) all possible itemsets lengths (under the apriori condition) are evaluated. verbose : int (default: 0) Shows the number of iterations if >= 1 and low_memory is True . If =1 and low_memory is False , shows the number of combinations. low_memory : bool (default: False) If True , uses an iterator to search for combinations above min_support . Note that while low_memory=True should only be used for large dataset if memory resources are limited, because this implementation is approx. 3-6x slower than the default. Returns pandas DataFrame with columns ['support', 'itemsets'] of all itemsets that are >= min_support and < than max_len (if max_len is not None). Each itemset in the 'itemsets' column is of type frozenset , which is a Python built-in type that behaves similarly to sets except that it is immutable (For more info, see https://docs.python.org/3.6/library/stdtypes.html#frozenset). Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/frequent_patterns/apriori/","title":"apriori"},{"location":"api_modules/mlxtend.frequent_patterns/association_rules/","text":"association_rules association_rules(df, metric='confidence', min_threshold=0.8, support_only=False) Generates a DataFrame of association rules including the metrics 'score', 'confidence', and 'lift' Parameters df : pandas DataFrame pandas DataFrame of frequent itemsets with columns ['support', 'itemsets'] metric : string (default: 'confidence') Metric to evaluate if a rule is of interest. Automatically set to 'support' if support_only=True . Otherwise, supported metrics are 'support', 'confidence', 'lift', 'leverage', 'conviction' and 'zhangs_metric' These metrics are computed as follows: - support(A->C) = support(A+C) [aka 'support'], range: [0, 1] - confidence(A->C) = support(A+C) / support(A), range: [0, 1] - lift(A->C) = confidence(A->C) / support(C), range: [0, inf] - leverage(A->C) = support(A->C) - support(A)*support(C), range: [-1, 1] - conviction = [1 - support(C)] / [1 - confidence(A->C)], range: [0, inf] - zhangs_metric(A->C) = leverage(A->C) / max(support(A->C)*(1-support(A)), support(A)*(support(C)-support(A->C))) range: [-1,1] min_threshold : float (default: 0.8) Minimal threshold for the evaluation metric, via the metric parameter, to decide whether a candidate rule is of interest. support_only : bool (default: False) Only computes the rule support and fills the other metric columns with NaNs. This is useful if: a) the input DataFrame is incomplete, e.g., does not contain support values for all rule antecedents and consequents b) you simply want to speed up the computation because you don't need the other metrics. Returns pandas DataFrame with columns \"antecedents\" and \"consequents\" that store itemsets, plus the scoring metric columns: \"antecedent support\", \"consequent support\", \"support\", \"confidence\", \"lift\", \"leverage\", \"conviction\" of all rules for which metric(rule) >= min_threshold. Each entry in the \"antecedents\" and \"consequents\" columns are of type frozenset , which is a Python built-in type that behaves similarly to sets except that it is immutable (For more info, see https://docs.python.org/3.6/library/stdtypes.html#frozenset). Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/frequent_patterns/association_rules/","title":"Association rules"},{"location":"api_modules/mlxtend.frequent_patterns/association_rules/#association_rules","text":"association_rules(df, metric='confidence', min_threshold=0.8, support_only=False) Generates a DataFrame of association rules including the metrics 'score', 'confidence', and 'lift' Parameters df : pandas DataFrame pandas DataFrame of frequent itemsets with columns ['support', 'itemsets'] metric : string (default: 'confidence') Metric to evaluate if a rule is of interest. Automatically set to 'support' if support_only=True . Otherwise, supported metrics are 'support', 'confidence', 'lift', 'leverage', 'conviction' and 'zhangs_metric' These metrics are computed as follows: - support(A->C) = support(A+C) [aka 'support'], range: [0, 1] - confidence(A->C) = support(A+C) / support(A), range: [0, 1] - lift(A->C) = confidence(A->C) / support(C), range: [0, inf] - leverage(A->C) = support(A->C) - support(A)*support(C), range: [-1, 1] - conviction = [1 - support(C)] / [1 - confidence(A->C)], range: [0, inf] - zhangs_metric(A->C) = leverage(A->C) / max(support(A->C)*(1-support(A)), support(A)*(support(C)-support(A->C))) range: [-1,1] min_threshold : float (default: 0.8) Minimal threshold for the evaluation metric, via the metric parameter, to decide whether a candidate rule is of interest. support_only : bool (default: False) Only computes the rule support and fills the other metric columns with NaNs. This is useful if: a) the input DataFrame is incomplete, e.g., does not contain support values for all rule antecedents and consequents b) you simply want to speed up the computation because you don't need the other metrics. Returns pandas DataFrame with columns \"antecedents\" and \"consequents\" that store itemsets, plus the scoring metric columns: \"antecedent support\", \"consequent support\", \"support\", \"confidence\", \"lift\", \"leverage\", \"conviction\" of all rules for which metric(rule) >= min_threshold. Each entry in the \"antecedents\" and \"consequents\" columns are of type frozenset , which is a Python built-in type that behaves similarly to sets except that it is immutable (For more info, see https://docs.python.org/3.6/library/stdtypes.html#frozenset). Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/frequent_patterns/association_rules/","title":"association_rules"},{"location":"api_modules/mlxtend.frequent_patterns/fpgrowth/","text":"fpgrowth fpgrowth(df, min_support=0.5, use_colnames=False, max_len=None, verbose=0) Get frequent itemsets from a one-hot DataFrame Parameters df : pandas DataFrame pandas DataFrame the encoded format. Also supports DataFrames with sparse data; for more info, please see https://pandas.pydata.org/pandas-docs/stable/user_guide/sparse.html#sparse-data-structures. Please note that the old pandas SparseDataFrame format is no longer supported in mlxtend >= 0.17.2. The allowed values are either 0/1 or True/False. For example, Apple Bananas Beer Chicken Milk Rice 0 True False True True False True 1 True False True False False True 2 True False True False False False 3 True True False False False False 4 False False True True True True 5 False False True False True True 6 False False True False True False 7 True True False False False False min_support : float (default: 0.5) A float between 0 and 1 for minimum support of the itemsets returned. The support is computed as the fraction transactions_where_item(s)_occur / total_transactions. use_colnames : bool (default: False) If true, uses the DataFrames' column names in the returned DataFrame instead of column indices. max_len : int (default: None) Maximum length of the itemsets generated. If None (default) all possible itemsets lengths are evaluated. verbose : int (default: 0) Shows the stages of conditional tree generation. Returns pandas DataFrame with columns ['support', 'itemsets'] of all itemsets that are >= min_support and < than max_len (if max_len is not None). Each itemset in the 'itemsets' column is of type frozenset , which is a Python built-in type that behaves similarly to sets except that it is immutable (For more info, see https://docs.python.org/3.6/library/stdtypes.html#frozenset). Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/frequent_patterns/fpgrowth/","title":"Fpgrowth"},{"location":"api_modules/mlxtend.frequent_patterns/fpgrowth/#fpgrowth","text":"fpgrowth(df, min_support=0.5, use_colnames=False, max_len=None, verbose=0) Get frequent itemsets from a one-hot DataFrame Parameters df : pandas DataFrame pandas DataFrame the encoded format. Also supports DataFrames with sparse data; for more info, please see https://pandas.pydata.org/pandas-docs/stable/user_guide/sparse.html#sparse-data-structures. Please note that the old pandas SparseDataFrame format is no longer supported in mlxtend >= 0.17.2. The allowed values are either 0/1 or True/False. For example, Apple Bananas Beer Chicken Milk Rice 0 True False True True False True 1 True False True False False True 2 True False True False False False 3 True True False False False False 4 False False True True True True 5 False False True False True True 6 False False True False True False 7 True True False False False False min_support : float (default: 0.5) A float between 0 and 1 for minimum support of the itemsets returned. The support is computed as the fraction transactions_where_item(s)_occur / total_transactions. use_colnames : bool (default: False) If true, uses the DataFrames' column names in the returned DataFrame instead of column indices. max_len : int (default: None) Maximum length of the itemsets generated. If None (default) all possible itemsets lengths are evaluated. verbose : int (default: 0) Shows the stages of conditional tree generation. Returns pandas DataFrame with columns ['support', 'itemsets'] of all itemsets that are >= min_support and < than max_len (if max_len is not None). Each itemset in the 'itemsets' column is of type frozenset , which is a Python built-in type that behaves similarly to sets except that it is immutable (For more info, see https://docs.python.org/3.6/library/stdtypes.html#frozenset). Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/frequent_patterns/fpgrowth/","title":"fpgrowth"},{"location":"api_modules/mlxtend.frequent_patterns/fpmax/","text":"fpmax fpmax(df, min_support=0.5, use_colnames=False, max_len=None, verbose=0) Get maximal frequent itemsets from a one-hot DataFrame Parameters df : pandas DataFrame pandas DataFrame the encoded format. Also supports DataFrames with sparse data; for more info, please see (https://pandas.pydata.org/pandas-docs/stable/ user_guide/sparse.html#sparse-data-structures) Please note that the old pandas SparseDataFrame format is no longer supported in mlxtend >= 0.17.2. The allowed values are either 0/1 or True/False. For example, Apple Bananas Beer Chicken Milk Rice 0 True False True True False True 1 True False True False False True 2 True False True False False False 3 True True False False False False 4 False False True True True True 5 False False True False True True 6 False False True False True False 7 True True False False False False min_support : float (default: 0.5) A float between 0 and 1 for minimum support of the itemsets returned. The support is computed as the fraction transactions_where_item(s)_occur / total_transactions. use_colnames : bool (default: False) If true, uses the DataFrames' column names in the returned DataFrame instead of column indices. max_len : int (default: None) Given the set of all maximal itemsets, return those that are less than max_len . If None (default) all possible itemsets lengths are evaluated. verbose : int (default: 0) Shows the stages of conditional tree generation. Returns pandas DataFrame with columns ['support', 'itemsets'] of all maximal itemsets that are >= min_support and < than max_len (if max_len is not None). Each itemset in the 'itemsets' column is of type frozenset , which is a Python built-in type that behaves similarly to sets except that it is immutable (For more info, see https://docs.python.org/3.6/library/stdtypes.html#frozenset). Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/frequent_patterns/fpmax/","title":"Fpmax"},{"location":"api_modules/mlxtend.frequent_patterns/fpmax/#fpmax","text":"fpmax(df, min_support=0.5, use_colnames=False, max_len=None, verbose=0) Get maximal frequent itemsets from a one-hot DataFrame Parameters df : pandas DataFrame pandas DataFrame the encoded format. Also supports DataFrames with sparse data; for more info, please see (https://pandas.pydata.org/pandas-docs/stable/ user_guide/sparse.html#sparse-data-structures) Please note that the old pandas SparseDataFrame format is no longer supported in mlxtend >= 0.17.2. The allowed values are either 0/1 or True/False. For example, Apple Bananas Beer Chicken Milk Rice 0 True False True True False True 1 True False True False False True 2 True False True False False False 3 True True False False False False 4 False False True True True True 5 False False True False True True 6 False False True False True False 7 True True False False False False min_support : float (default: 0.5) A float between 0 and 1 for minimum support of the itemsets returned. The support is computed as the fraction transactions_where_item(s)_occur / total_transactions. use_colnames : bool (default: False) If true, uses the DataFrames' column names in the returned DataFrame instead of column indices. max_len : int (default: None) Given the set of all maximal itemsets, return those that are less than max_len . If None (default) all possible itemsets lengths are evaluated. verbose : int (default: 0) Shows the stages of conditional tree generation. Returns pandas DataFrame with columns ['support', 'itemsets'] of all maximal itemsets that are >= min_support and < than max_len (if max_len is not None). Each itemset in the 'itemsets' column is of type frozenset , which is a Python built-in type that behaves similarly to sets except that it is immutable (For more info, see https://docs.python.org/3.6/library/stdtypes.html#frozenset). Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/frequent_patterns/fpmax/","title":"fpmax"},{"location":"api_modules/mlxtend.frequent_patterns/hmine/","text":"hmine hmine(df, min_support=0.5, use_colnames=False, max_len=None, verbose=0) -> pandas.core.frame.DataFrame Get frequent itemsets from a one-hot DataFrame Parameters df : pandas DataFrame pandas DataFrame the encoded format. Also supports DataFrames with sparse data; for more info, please see https://pandas.pydata.org/pandas-docs/stable/user_guide/sparse.html#sparse-data-structures. Please note that the old pandas SparseDataFrame format is no longer supported in mlxtend >= 0.17.2. The allowed values are either 0/1 or True/False. For example, Apple Bananas Beer Chicken Milk Rice 0 True False True True False True 1 True False True False False True 2 True False True False False False 3 True True False False False False 4 False False True True True True 5 False False True False True True 6 False False True False True False 7 True True False False False False min_support : float (default: 0.5) A float between 0 and 1 for minimum support of the itemsets returned. The support is computed as the fraction transactions_where_item(s)_occur / total_transactions. use_colnames : bool (default: False) If true, uses the DataFrames' column names in the returned DataFrame instead of column indices. max_len : int (default: None) Maximum length of the itemsets generated. If None (default) all possible itemsets lengths are evaluated. verbose : int (default: 0) Shows the stages of conditional tree generation. Returns pandas DataFrame with columns ['support', 'itemsets'] of all itemsets that are >= min_support and < than max_len (if max_len is not None). Each itemset in the 'itemsets' column is of type frozenset , which is a Python built-in type that behaves similarly to sets except that it is immutable (For more info, see https://docs.python.org/3.6/library/stdtypes.html#frozenset). Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/frequent_patterns/hmine/","title":"Hmine"},{"location":"api_modules/mlxtend.frequent_patterns/hmine/#hmine","text":"hmine(df, min_support=0.5, use_colnames=False, max_len=None, verbose=0) -> pandas.core.frame.DataFrame Get frequent itemsets from a one-hot DataFrame Parameters df : pandas DataFrame pandas DataFrame the encoded format. Also supports DataFrames with sparse data; for more info, please see https://pandas.pydata.org/pandas-docs/stable/user_guide/sparse.html#sparse-data-structures. Please note that the old pandas SparseDataFrame format is no longer supported in mlxtend >= 0.17.2. The allowed values are either 0/1 or True/False. For example, Apple Bananas Beer Chicken Milk Rice 0 True False True True False True 1 True False True False False True 2 True False True False False False 3 True True False False False False 4 False False True True True True 5 False False True False True True 6 False False True False True False 7 True True False False False False min_support : float (default: 0.5) A float between 0 and 1 for minimum support of the itemsets returned. The support is computed as the fraction transactions_where_item(s)_occur / total_transactions. use_colnames : bool (default: False) If true, uses the DataFrames' column names in the returned DataFrame instead of column indices. max_len : int (default: None) Maximum length of the itemsets generated. If None (default) all possible itemsets lengths are evaluated. verbose : int (default: 0) Shows the stages of conditional tree generation. Returns pandas DataFrame with columns ['support', 'itemsets'] of all itemsets that are >= min_support and < than max_len (if max_len is not None). Each itemset in the 'itemsets' column is of type frozenset , which is a Python built-in type that behaves similarly to sets except that it is immutable (For more info, see https://docs.python.org/3.6/library/stdtypes.html#frozenset). Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/frequent_patterns/hmine/","title":"hmine"},{"location":"api_modules/mlxtend.image/EyepadAlign/","text":"EyepadAlign EyepadAlign(verbose=0) Class to align/transform face images to facial landmarks, based on eye alignment. 1. A scaling factor is computed based on distance between the left and right eye, such that the transformed face image will have the same eye distance as a reference face image. 2. A transformation is performed based on the eyes' center point. to align the face based on the reference eye location. 3. Finally, the transformed image is padded with zeros to match the desired final image size. Parameters verbose : int (default=0) Verbose level to display the progress bar and log messages. Setting verbose=1 will print a progress bar upon calling fit_directory . Attributes target_landmarks_ : target landmarks to transform new face images to. Depending on the chosen fit parameters, it can be either (1) assigned to pre-fit shapes, (2) computed from a single face image (3) computed as the mean of face landmarks from all face images in a file directory of face images. eye_distance_ : the distance between left and right eyes in the target landmarks. target_height_ : the height of the transformed output image. target_width_ : the width of the transformed output image. Returns self : object Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/image/eyepad_align/ Methods fit_directory(target_img_dir, target_height, target_width, file_extension='.jpg', pre_check=True) Calculates the average landmarks for all face images in a directory which will then be set as the target landmark set. Arguments target_img_dir : str Directory containing the images target_height : int Expected image height of the images in the directory target_width : int Expected image width of the images in the directory file_extension str (default='.jpg'): File extension of the image files. pre_check Bool (default=True): Checks that each image has the dimensions specificed via target_height and target_width on the whole directory first to identify potential issues that are recommended to be fixed before proceeding. Raises a warning for each image if dimensions differ from the ones specified and expected. Returns self : object fit_image(target_image) Derives facial landmarks from a target image. Arguments target_image : uint8 numpy.array, shape=[width, height, channels] NumPy array representation of the image data. Returns self : object fit_values(target_landmarks, target_width, target_height) Used for determining the eye location from pre-defined landmark arrays, eliminating the need for re-computing the average landmarks on a target image or image directory. Arguments target_landmarks : np.array, shape=(height, width) NumPy array containing the locations of the facial landmarks as determined by mlxtend.image.extract_face_landmarks target_height : int image height target_width : int image width Returns self : object transform(img) transforms a single face image (img) to the target landmarks based on the location of the eyes by scaling, translation and cropping (if needed): (1) Scaling the image so that the distance of the two eyes in the given image (img) matches the distance of the two eyes in the target landmarks. (2) Translation is performed based on the middle point between the two eyes. Arguments img : np.array, shape=(height, width, channels) Input image to be transformed. Returns self : object","title":"EyepadAlign"},{"location":"api_modules/mlxtend.image/EyepadAlign/#eyepadalign","text":"EyepadAlign(verbose=0) Class to align/transform face images to facial landmarks, based on eye alignment. 1. A scaling factor is computed based on distance between the left and right eye, such that the transformed face image will have the same eye distance as a reference face image. 2. A transformation is performed based on the eyes' center point. to align the face based on the reference eye location. 3. Finally, the transformed image is padded with zeros to match the desired final image size. Parameters verbose : int (default=0) Verbose level to display the progress bar and log messages. Setting verbose=1 will print a progress bar upon calling fit_directory . Attributes target_landmarks_ : target landmarks to transform new face images to. Depending on the chosen fit parameters, it can be either (1) assigned to pre-fit shapes, (2) computed from a single face image (3) computed as the mean of face landmarks from all face images in a file directory of face images. eye_distance_ : the distance between left and right eyes in the target landmarks. target_height_ : the height of the transformed output image. target_width_ : the width of the transformed output image. Returns self : object Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/image/eyepad_align/","title":"EyepadAlign"},{"location":"api_modules/mlxtend.image/EyepadAlign/#methods","text":"fit_directory(target_img_dir, target_height, target_width, file_extension='.jpg', pre_check=True) Calculates the average landmarks for all face images in a directory which will then be set as the target landmark set. Arguments target_img_dir : str Directory containing the images target_height : int Expected image height of the images in the directory target_width : int Expected image width of the images in the directory file_extension str (default='.jpg'): File extension of the image files. pre_check Bool (default=True): Checks that each image has the dimensions specificed via target_height and target_width on the whole directory first to identify potential issues that are recommended to be fixed before proceeding. Raises a warning for each image if dimensions differ from the ones specified and expected. Returns self : object fit_image(target_image) Derives facial landmarks from a target image. Arguments target_image : uint8 numpy.array, shape=[width, height, channels] NumPy array representation of the image data. Returns self : object fit_values(target_landmarks, target_width, target_height) Used for determining the eye location from pre-defined landmark arrays, eliminating the need for re-computing the average landmarks on a target image or image directory. Arguments target_landmarks : np.array, shape=(height, width) NumPy array containing the locations of the facial landmarks as determined by mlxtend.image.extract_face_landmarks target_height : int image height target_width : int image width Returns self : object transform(img) transforms a single face image (img) to the target landmarks based on the location of the eyes by scaling, translation and cropping (if needed): (1) Scaling the image so that the distance of the two eyes in the given image (img) matches the distance of the two eyes in the target landmarks. (2) Translation is performed based on the middle point between the two eyes. Arguments img : np.array, shape=(height, width, channels) Input image to be transformed. Returns self : object","title":"Methods"},{"location":"api_modules/mlxtend.image/extract_face_landmarks/","text":"extract_face_landmarks extract_face_landmarks(img, return_dtype= ) Function to extract face landmarks. Note that this function requires an installation of the Python version of the library \"dlib\": https://dlib.net Parameters img : array, shape = [h, w, ?] Numpy array of a face image or imageio.core.util.Array. E.g., img = imageio.core.util.Array(ary) Supported shapes are - 3D tensors with 1 or more color channels, for example, RGB: [h, w, 3] - 2D tensors without color channel, for example, Grayscale: [h, w] return_dtype: the return data-type of the array, default: np.int32. Returns landmarks : numpy.ndarray, shape = [68, 2] A numpy array, where each row contains a landmark/point x-y coordinates. Return None if no face is detected by Dlib. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/image/extract_face_landmarks/","title":"Extract face landmarks"},{"location":"api_modules/mlxtend.image/extract_face_landmarks/#extract_face_landmarks","text":"extract_face_landmarks(img, return_dtype= ) Function to extract face landmarks. Note that this function requires an installation of the Python version of the library \"dlib\": https://dlib.net Parameters img : array, shape = [h, w, ?] Numpy array of a face image or imageio.core.util.Array. E.g., img = imageio.core.util.Array(ary) Supported shapes are - 3D tensors with 1 or more color channels, for example, RGB: [h, w, 3] - 2D tensors without color channel, for example, Grayscale: [h, w] return_dtype: the return data-type of the array, default: np.int32. Returns landmarks : numpy.ndarray, shape = [68, 2] A numpy array, where each row contains a landmark/point x-y coordinates. Return None if no face is detected by Dlib. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/image/extract_face_landmarks/","title":"extract_face_landmarks"},{"location":"api_modules/mlxtend.math/factorial/","text":"factorial factorial(n) None","title":"Factorial"},{"location":"api_modules/mlxtend.math/factorial/#factorial","text":"factorial(n) None","title":"factorial"},{"location":"api_modules/mlxtend.math/num_combinations/","text":"num_combinations num_combinations(n, k, with_replacement=False) Function to calculate the number of possible combinations. Parameters n : int Total number of items. k : int Number of elements of the target itemset. with_replacement : bool (default: False) Allows repeated elements if True. Returns comb : int Number of possible combinations. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/math/num_combinations/","title":"Num combinations"},{"location":"api_modules/mlxtend.math/num_combinations/#num_combinations","text":"num_combinations(n, k, with_replacement=False) Function to calculate the number of possible combinations. Parameters n : int Total number of items. k : int Number of elements of the target itemset. with_replacement : bool (default: False) Allows repeated elements if True. Returns comb : int Number of possible combinations. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/math/num_combinations/","title":"num_combinations"},{"location":"api_modules/mlxtend.math/num_permutations/","text":"num_permutations num_permutations(n, k, with_replacement=False) Function to calculate the number of possible permutations. Parameters n : int Total number of items. k : int Number of elements of the target itemset. with_replacement : bool Allows repeated elements if True. Returns permut : int Number of possible permutations. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/math/num_permutations/","title":"Num permutations"},{"location":"api_modules/mlxtend.math/num_permutations/#num_permutations","text":"num_permutations(n, k, with_replacement=False) Function to calculate the number of possible permutations. Parameters n : int Total number of items. k : int Number of elements of the target itemset. with_replacement : bool Allows repeated elements if True. Returns permut : int Number of possible permutations. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/math/num_permutations/","title":"num_permutations"},{"location":"api_modules/mlxtend.math/vectorspace_dimensionality/","text":"vectorspace_dimensionality vectorspace_dimensionality(ary) Computes the hyper-volume spanned by a vector set Parameters ary : array-like, shape=[num_vectors, num_vectors] An orthogonal set of vectors (arranged as columns in a matrix) Returns dimensions : int An integer indicating the \"dimensionality\" hyper-volume spanned by the vector set Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/math/vectorspace_dimensionality/","title":"Vectorspace dimensionality"},{"location":"api_modules/mlxtend.math/vectorspace_dimensionality/#vectorspace_dimensionality","text":"vectorspace_dimensionality(ary) Computes the hyper-volume spanned by a vector set Parameters ary : array-like, shape=[num_vectors, num_vectors] An orthogonal set of vectors (arranged as columns in a matrix) Returns dimensions : int An integer indicating the \"dimensionality\" hyper-volume spanned by the vector set Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/math/vectorspace_dimensionality/","title":"vectorspace_dimensionality"},{"location":"api_modules/mlxtend.math/vectorspace_orthonormalization/","text":"vectorspace_orthonormalization vectorspace_orthonormalization(ary, eps=1e-13) Transforms a set of column vectors to a orthonormal basis. Given a set of orthogonal vectors, this functions converts such column vectors, arranged in a matrix, into orthonormal basis vectors. Parameters ary : array-like, shape=[num_vectors, num_vectors] An orthogonal set of vectors (arranged as columns in a matrix) eps : float (default: 1e-13) A small tolerance value to determine whether the vector norm is zero or not. Returns arr : array-like, shape=[num_vectors, num_vectors] An orthonormal set of vectors (arranged as columns) Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/math/vectorspace_orthonormalization/","title":"Vectorspace orthonormalization"},{"location":"api_modules/mlxtend.math/vectorspace_orthonormalization/#vectorspace_orthonormalization","text":"vectorspace_orthonormalization(ary, eps=1e-13) Transforms a set of column vectors to a orthonormal basis. Given a set of orthogonal vectors, this functions converts such column vectors, arranged in a matrix, into orthonormal basis vectors. Parameters ary : array-like, shape=[num_vectors, num_vectors] An orthogonal set of vectors (arranged as columns in a matrix) eps : float (default: 1e-13) A small tolerance value to determine whether the vector norm is zero or not. Returns arr : array-like, shape=[num_vectors, num_vectors] An orthonormal set of vectors (arranged as columns) Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/math/vectorspace_orthonormalization/","title":"vectorspace_orthonormalization"},{"location":"api_modules/mlxtend.plotting/category_scatter/","text":"category_scatter category_scatter(x, y, label_col, data, markers='sxo^v', colors=('blue', 'green', 'red', 'purple', 'gray', 'cyan'), alpha=0.7, markersize=20.0, legend_loc='best') Scatter plot to plot categories in different colors/markerstyles. Parameters x : str or int DataFrame column name of the x-axis values or integer for the numpy ndarray column index. y : str DataFrame column name of the y-axis values or integer for the numpy ndarray column index data : Pandas DataFrame object or NumPy ndarray. markers : str Markers that are cycled through the label category. colors : tuple Colors that are cycled through the label category. alpha : float (default: 0.7) Parameter to control the transparency. markersize : float (default` : 20.0) Parameter to control the marker size. legend_loc : str (default: 'best') Location of the plot legend {best, upper left, upper right, lower left, lower right} No legend if legend_loc=False Returns fig : matplotlig.pyplot figure object Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/plotting/category_scatter/","title":"Category scatter"},{"location":"api_modules/mlxtend.plotting/category_scatter/#category_scatter","text":"category_scatter(x, y, label_col, data, markers='sxo^v', colors=('blue', 'green', 'red', 'purple', 'gray', 'cyan'), alpha=0.7, markersize=20.0, legend_loc='best') Scatter plot to plot categories in different colors/markerstyles. Parameters x : str or int DataFrame column name of the x-axis values or integer for the numpy ndarray column index. y : str DataFrame column name of the y-axis values or integer for the numpy ndarray column index data : Pandas DataFrame object or NumPy ndarray. markers : str Markers that are cycled through the label category. colors : tuple Colors that are cycled through the label category. alpha : float (default: 0.7) Parameter to control the transparency. markersize : float (default` : 20.0) Parameter to control the marker size. legend_loc : str (default: 'best') Location of the plot legend {best, upper left, upper right, lower left, lower right} No legend if legend_loc=False Returns fig : matplotlig.pyplot figure object Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/plotting/category_scatter/","title":"category_scatter"},{"location":"api_modules/mlxtend.plotting/checkerboard_plot/","text":"checkerboard_plot checkerboard_plot(ary, cell_colors=('white', 'black'), font_colors=('black', 'white'), fmt='%.1f', figsize=None, row_labels=None, col_labels=None, fontsize=None) Plot a checkerboard table / heatmap via matplotlib. Parameters ary : array-like, shape = [n, m] A 2D Nnumpy array. cell_colors : tuple or list (default: ('white', 'black')) Tuple or list containing the two colors of the checkerboard pattern. font_colors : tuple or list (default: ('black', 'white')) Font colors corresponding to the cell colors. figsize : tuple (default: (2.5, 2.5)) Height and width of the figure fmt : str (default: '%.1f') Python string formatter for cell values. The default '%.1f' results in floats with 1 digit after the decimal point. Use '%d' to show numbers as integers. row_labels : list (default: None) List of the row labels. Uses the array row indices 0 to n by default. col_labels : list (default: None) List of the column labels. Uses the array column indices 0 to m by default. fontsize : int (default: None) Specifies the font size of the checkerboard table. Uses matplotlib's default if None. Returns fig : matplotlib Figure object. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/plotting/checkerboard_plot/","title":"Checkerboard plot"},{"location":"api_modules/mlxtend.plotting/checkerboard_plot/#checkerboard_plot","text":"checkerboard_plot(ary, cell_colors=('white', 'black'), font_colors=('black', 'white'), fmt='%.1f', figsize=None, row_labels=None, col_labels=None, fontsize=None) Plot a checkerboard table / heatmap via matplotlib. Parameters ary : array-like, shape = [n, m] A 2D Nnumpy array. cell_colors : tuple or list (default: ('white', 'black')) Tuple or list containing the two colors of the checkerboard pattern. font_colors : tuple or list (default: ('black', 'white')) Font colors corresponding to the cell colors. figsize : tuple (default: (2.5, 2.5)) Height and width of the figure fmt : str (default: '%.1f') Python string formatter for cell values. The default '%.1f' results in floats with 1 digit after the decimal point. Use '%d' to show numbers as integers. row_labels : list (default: None) List of the row labels. Uses the array row indices 0 to n by default. col_labels : list (default: None) List of the column labels. Uses the array column indices 0 to m by default. fontsize : int (default: None) Specifies the font size of the checkerboard table. Uses matplotlib's default if None. Returns fig : matplotlib Figure object. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/plotting/checkerboard_plot/","title":"checkerboard_plot"},{"location":"api_modules/mlxtend.plotting/ecdf/","text":"ecdf ecdf(x, y_label='ECDF', x_label=None, ax=None, percentile=None, ecdf_color=None, ecdf_marker='o', percentile_color='black', percentile_linestyle='--') Plots an Empirical Cumulative Distribution Function Parameters x : array or list, shape=[n_samples,] Array-like object containing the feature values y_label : str (default='ECDF') Text label for the y-axis x_label : str (default=None) Text label for the x-axis ax : matplotlib.axes.Axes (default: None) An existing matplotlib Axes. Creates one if ax=None percentile : float (default=None) Float between 0 and 1 for plotting a percentile threshold line ecdf_color : matplotlib color (default=None) Color for the ECDF plot; uses matplotlib defaults if None ecdf_marker : matplotlib marker (default='o') Marker style for the ECDF plot percentile_color : matplotlib color (default='black') Color for the percentile threshold if percentile is not None percentile_linestyle : matplotlib linestyle (default='--') Line style for the percentile threshold if percentile is not None Returns ax : matplotlib.axes.Axes object percentile_threshold : float Feature threshold at the percentile or None if percentile=None percentile_count : Number of if percentile is not None Number of samples that have a feature less or equal than the feature threshold at a percentile threshold or None if percentile=None Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/plotting/ecdf/","title":"Ecdf"},{"location":"api_modules/mlxtend.plotting/ecdf/#ecdf","text":"ecdf(x, y_label='ECDF', x_label=None, ax=None, percentile=None, ecdf_color=None, ecdf_marker='o', percentile_color='black', percentile_linestyle='--') Plots an Empirical Cumulative Distribution Function Parameters x : array or list, shape=[n_samples,] Array-like object containing the feature values y_label : str (default='ECDF') Text label for the y-axis x_label : str (default=None) Text label for the x-axis ax : matplotlib.axes.Axes (default: None) An existing matplotlib Axes. Creates one if ax=None percentile : float (default=None) Float between 0 and 1 for plotting a percentile threshold line ecdf_color : matplotlib color (default=None) Color for the ECDF plot; uses matplotlib defaults if None ecdf_marker : matplotlib marker (default='o') Marker style for the ECDF plot percentile_color : matplotlib color (default='black') Color for the percentile threshold if percentile is not None percentile_linestyle : matplotlib linestyle (default='--') Line style for the percentile threshold if percentile is not None Returns ax : matplotlib.axes.Axes object percentile_threshold : float Feature threshold at the percentile or None if percentile=None percentile_count : Number of if percentile is not None Number of samples that have a feature less or equal than the feature threshold at a percentile threshold or None if percentile=None Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/plotting/ecdf/","title":"ecdf"},{"location":"api_modules/mlxtend.plotting/enrichment_plot/","text":"enrichment_plot enrichment_plot(df, colors='bgrkcy', markers=' ', linestyles='-', alpha=0.5, lw=2, where='post', grid=True, count_label='Count', xlim='auto', ylim='auto', invert_axes=False, legend_loc='best', ax=None) Plot stacked barplots Parameters df : pandas.DataFrame A pandas DataFrame where columns represent the different categories. colors: str (default: 'bgrcky') The colors of the bars. markers : str (default: ' ') Matplotlib markerstyles, e.g, 'sov' for square,circle, and triangle markers. linestyles : str (default: '-') Matplotlib linestyles, e.g., '-,--' to cycle normal and dashed lines. Note that the different linestyles need to be separated by commas. alpha : float (default: 0.5) Transparency level from 0.0 to 1.0. lw : int or float (default: 2) Linewidth parameter. where : {'post', 'pre', 'mid'} (default: 'post') Starting location of the steps. grid : bool (default: True ) Plots a grid if True. count_label : str (default: 'Count') Label for the \"Count\"-axis. xlim : 'auto' or array-like [min, max] (default: 'auto') Min and maximum position of the x-axis range. ylim : 'auto' or array-like [min, max] (default: 'auto') Min and maximum position of the y-axis range. invert_axes : bool (default: False) Plots count on the x-axis if True. legend_loc : str (default: 'best') Location of the plot legend {best, upper left, upper right, lower left, lower right} No legend if legend_loc=False ax : matplotlib axis, optional (default: None) Use this axis for plotting or make a new one otherwise Returns ax : matplotlib axis Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/plotting/enrichment_plot/","title":"Enrichment plot"},{"location":"api_modules/mlxtend.plotting/enrichment_plot/#enrichment_plot","text":"enrichment_plot(df, colors='bgrkcy', markers=' ', linestyles='-', alpha=0.5, lw=2, where='post', grid=True, count_label='Count', xlim='auto', ylim='auto', invert_axes=False, legend_loc='best', ax=None) Plot stacked barplots Parameters df : pandas.DataFrame A pandas DataFrame where columns represent the different categories. colors: str (default: 'bgrcky') The colors of the bars. markers : str (default: ' ') Matplotlib markerstyles, e.g, 'sov' for square,circle, and triangle markers. linestyles : str (default: '-') Matplotlib linestyles, e.g., '-,--' to cycle normal and dashed lines. Note that the different linestyles need to be separated by commas. alpha : float (default: 0.5) Transparency level from 0.0 to 1.0. lw : int or float (default: 2) Linewidth parameter. where : {'post', 'pre', 'mid'} (default: 'post') Starting location of the steps. grid : bool (default: True ) Plots a grid if True. count_label : str (default: 'Count') Label for the \"Count\"-axis. xlim : 'auto' or array-like [min, max] (default: 'auto') Min and maximum position of the x-axis range. ylim : 'auto' or array-like [min, max] (default: 'auto') Min and maximum position of the y-axis range. invert_axes : bool (default: False) Plots count on the x-axis if True. legend_loc : str (default: 'best') Location of the plot legend {best, upper left, upper right, lower left, lower right} No legend if legend_loc=False ax : matplotlib axis, optional (default: None) Use this axis for plotting or make a new one otherwise Returns ax : matplotlib axis Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/plotting/enrichment_plot/","title":"enrichment_plot"},{"location":"api_modules/mlxtend.plotting/heatmap/","text":"heatmap heatmap(matrix, hide_spines=False, hide_ticks=False, figsize=None, cmap=None, colorbar=True, row_names=None, column_names=None, column_name_rotation=45, cell_values=True, cell_fmt='.2f', cell_font_size=None, text_color_threshold=None) Plot a heatmap via matplotlib. Parameters conf_mat : array-like, shape = [n_rows, n_columns] And arbitrary 2D array. hide_spines : bool (default: False) Hides axis spines if True. hide_ticks : bool (default: False) Hides axis ticks if True figsize : tuple (default: (2.5, 2.5)) Height and width of the figure cmap : matplotlib colormap (default: None ) Uses matplotlib.pyplot.cm.viridis if None colorbar : bool (default: True) Shows a colorbar if True row_names : array-like, shape = [n_rows] (default: None) List of row names to be used as y-axis tick labels. column_names : array-like, shape = [n_columns] (default: None) List of column names to be used as x-axis tick labels. column_name_rotation : int (default: 45) Number of degrees for rotating column x-tick labels. cell_values : bool (default: True) Plots cell values if True. cell_fmt : string (default: '.2f') Format specification for cell values (if cell_values=True ) cell_font_size : int (default: None) Font size for cell values (if cell_values=True ) text_color_threshold : float (default: None) Threshold for the black/white text threshold of the text annotation. Default (None) tried to infer a good threshold automatically using np.max(normed_matrix) / 2 . Returns fig, ax : matplotlib.pyplot subplot objects Figure and axis elements of the subplot. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/plotting/heatmap/","title":"Heatmap"},{"location":"api_modules/mlxtend.plotting/heatmap/#heatmap","text":"heatmap(matrix, hide_spines=False, hide_ticks=False, figsize=None, cmap=None, colorbar=True, row_names=None, column_names=None, column_name_rotation=45, cell_values=True, cell_fmt='.2f', cell_font_size=None, text_color_threshold=None) Plot a heatmap via matplotlib. Parameters conf_mat : array-like, shape = [n_rows, n_columns] And arbitrary 2D array. hide_spines : bool (default: False) Hides axis spines if True. hide_ticks : bool (default: False) Hides axis ticks if True figsize : tuple (default: (2.5, 2.5)) Height and width of the figure cmap : matplotlib colormap (default: None ) Uses matplotlib.pyplot.cm.viridis if None colorbar : bool (default: True) Shows a colorbar if True row_names : array-like, shape = [n_rows] (default: None) List of row names to be used as y-axis tick labels. column_names : array-like, shape = [n_columns] (default: None) List of column names to be used as x-axis tick labels. column_name_rotation : int (default: 45) Number of degrees for rotating column x-tick labels. cell_values : bool (default: True) Plots cell values if True. cell_fmt : string (default: '.2f') Format specification for cell values (if cell_values=True ) cell_font_size : int (default: None) Font size for cell values (if cell_values=True ) text_color_threshold : float (default: None) Threshold for the black/white text threshold of the text annotation. Default (None) tried to infer a good threshold automatically using np.max(normed_matrix) / 2 . Returns fig, ax : matplotlib.pyplot subplot objects Figure and axis elements of the subplot. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/plotting/heatmap/","title":"heatmap"},{"location":"api_modules/mlxtend.plotting/plot_confusion_matrix/","text":"plot_confusion_matrix plot_confusion_matrix(conf_mat, hide_spines=False, hide_ticks=False, figsize=None, cmap=None, colorbar=False, show_absolute=True, show_normed=False, norm_colormap=None, class_names=None, figure=None, axis=None, fontcolor_threshold=0.5) Plot a confusion matrix via matplotlib. Parameters conf_mat : array-like, shape = [n_classes, n_classes] Confusion matrix from evaluate.confusion matrix. hide_spines : bool (default: False) Hides axis spines if True. hide_ticks : bool (default: False) Hides axis ticks if True figsize : tuple (default: (2.5, 2.5)) Height and width of the figure cmap : matplotlib colormap (default: None ) Uses matplotlib.pyplot.cm.Blues if None colorbar : bool (default: False) Shows a colorbar if True show_absolute : bool (default: True) Shows absolute confusion matrix coefficients if True. At least one of show_absolute or show_normed must be True. show_normed : bool (default: False) Shows normed confusion matrix coefficients if True. The normed confusion matrix coefficients give the proportion of training examples per class that are assigned the correct label. At least one of show_absolute or show_normed must be True. norm_colormap : bool (default: False) Matplotlib color normalization object to normalize the color scale, e.g., matplotlib.colors.LogNorm() . class_names : array-like, shape = [n_classes] (default: None) List of class names. If not None , ticks will be set to these values. figure : None or Matplotlib figure (default: None) If None will create a new figure. axis : None or Matplotlib figure axis (default: None) If None will create a new axis. fontcolor_threshold : Float (default: 0.5) Sets a threshold for choosing black and white font colors for the cells. By default all values larger than 0.5 times the maximum cell value are converted to white, and everything equal or smaller than 0.5 times the maximum cell value are converted to black. Returns fig, ax : matplotlib.pyplot subplot objects Figure and axis elements of the subplot. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/plotting/plot_confusion_matrix/","title":"Plot confusion matrix"},{"location":"api_modules/mlxtend.plotting/plot_confusion_matrix/#plot_confusion_matrix","text":"plot_confusion_matrix(conf_mat, hide_spines=False, hide_ticks=False, figsize=None, cmap=None, colorbar=False, show_absolute=True, show_normed=False, norm_colormap=None, class_names=None, figure=None, axis=None, fontcolor_threshold=0.5) Plot a confusion matrix via matplotlib. Parameters conf_mat : array-like, shape = [n_classes, n_classes] Confusion matrix from evaluate.confusion matrix. hide_spines : bool (default: False) Hides axis spines if True. hide_ticks : bool (default: False) Hides axis ticks if True figsize : tuple (default: (2.5, 2.5)) Height and width of the figure cmap : matplotlib colormap (default: None ) Uses matplotlib.pyplot.cm.Blues if None colorbar : bool (default: False) Shows a colorbar if True show_absolute : bool (default: True) Shows absolute confusion matrix coefficients if True. At least one of show_absolute or show_normed must be True. show_normed : bool (default: False) Shows normed confusion matrix coefficients if True. The normed confusion matrix coefficients give the proportion of training examples per class that are assigned the correct label. At least one of show_absolute or show_normed must be True. norm_colormap : bool (default: False) Matplotlib color normalization object to normalize the color scale, e.g., matplotlib.colors.LogNorm() . class_names : array-like, shape = [n_classes] (default: None) List of class names. If not None , ticks will be set to these values. figure : None or Matplotlib figure (default: None) If None will create a new figure. axis : None or Matplotlib figure axis (default: None) If None will create a new axis. fontcolor_threshold : Float (default: 0.5) Sets a threshold for choosing black and white font colors for the cells. By default all values larger than 0.5 times the maximum cell value are converted to white, and everything equal or smaller than 0.5 times the maximum cell value are converted to black. Returns fig, ax : matplotlib.pyplot subplot objects Figure and axis elements of the subplot. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/plotting/plot_confusion_matrix/","title":"plot_confusion_matrix"},{"location":"api_modules/mlxtend.plotting/plot_decision_regions/","text":"plot_decision_regions plot_decision_regions(X, y, clf, feature_index=None, filler_feature_values=None, filler_feature_ranges=None, ax=None, X_highlight=None, zoom_factor=1.0, legend=1, hide_spines=True, markers='s^oxv<>', colors='#1f77b4,#ff7f0e,#3ca02c,#d62728,#9467bd,#8c564b,#e377c2,#7f7f7f,#bcbd22,#17becf', scatter_kwargs=None, contourf_kwargs=None, contour_kwargs=None, scatter_highlight_kwargs=None, n_jobs=None) Plot decision regions of a classifier. Please note that this functions assumes that class labels are labeled consecutively, e.g,. 0, 1, 2, 3, 4, and 5. If you have class labels with integer labels > 4, you may want to provide additional colors and/or markers as `colors` and `markers` arguments. See https://matplotlib.org/examples/color/named_colors.html for more information. Parameters X : array-like, shape = [n_samples, n_features] Feature Matrix. y : array-like, shape = [n_samples] True class labels. clf : Classifier object. Must have a .predict method. feature_index : array-like (default: (0,) for 1D, (0, 1) otherwise) Feature indices to use for plotting. The first index in feature_index will be on the x-axis, the second index will be on the y-axis. filler_feature_values : dict (default: None) Only needed for number features > 2. Dictionary of feature index-value pairs for the features not being plotted. filler_feature_ranges : dict (default: None) Only needed for number features > 2. Dictionary of feature index-value pairs for the features not being plotted. Will use the ranges provided to select training samples for plotting. ax : matplotlib.axes.Axes (default: None) An existing matplotlib Axes. Creates one if ax=None. X_highlight : array-like, shape = [n_samples, n_features] (default: None) An array with data points that are used to highlight samples in X . zoom_factor : float (default: 1.0) Controls the scale of the x- and y-axis of the decision plot. hide_spines : bool (default: True) Hide axis spines if True. legend : int (default: 1) Integer to specify the legend location. No legend if legend is 0. markers : str (default: 's^oxv<>') Scatterplot markers. colors : str (default: 'red,blue,limegreen,gray,cyan') Comma separated list of colors. scatter_kwargs : dict (default: None) Keyword arguments for underlying matplotlib scatter function. contourf_kwargs : dict (default: None) Keyword arguments for underlying matplotlib contourf function. contour_kwargs : dict (default: None) Keyword arguments for underlying matplotlib contour function (which draws the lines between decision regions). scatter_highlight_kwargs : dict (default: None) Keyword arguments for underlying matplotlib scatter function. n_jobs : int or None, optional (default=None) The number of CPUs to use to do the computation using Python's multiprocessing library. None means 1. -1 means using all processors. New in v0.22.0. Returns ax : matplotlib.axes.Axes object Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/plotting/plot_decision_regions/","title":"Plot decision regions"},{"location":"api_modules/mlxtend.plotting/plot_decision_regions/#plot_decision_regions","text":"plot_decision_regions(X, y, clf, feature_index=None, filler_feature_values=None, filler_feature_ranges=None, ax=None, X_highlight=None, zoom_factor=1.0, legend=1, hide_spines=True, markers='s^oxv<>', colors='#1f77b4,#ff7f0e,#3ca02c,#d62728,#9467bd,#8c564b,#e377c2,#7f7f7f,#bcbd22,#17becf', scatter_kwargs=None, contourf_kwargs=None, contour_kwargs=None, scatter_highlight_kwargs=None, n_jobs=None) Plot decision regions of a classifier. Please note that this functions assumes that class labels are labeled consecutively, e.g,. 0, 1, 2, 3, 4, and 5. If you have class labels with integer labels > 4, you may want to provide additional colors and/or markers as `colors` and `markers` arguments. See https://matplotlib.org/examples/color/named_colors.html for more information. Parameters X : array-like, shape = [n_samples, n_features] Feature Matrix. y : array-like, shape = [n_samples] True class labels. clf : Classifier object. Must have a .predict method. feature_index : array-like (default: (0,) for 1D, (0, 1) otherwise) Feature indices to use for plotting. The first index in feature_index will be on the x-axis, the second index will be on the y-axis. filler_feature_values : dict (default: None) Only needed for number features > 2. Dictionary of feature index-value pairs for the features not being plotted. filler_feature_ranges : dict (default: None) Only needed for number features > 2. Dictionary of feature index-value pairs for the features not being plotted. Will use the ranges provided to select training samples for plotting. ax : matplotlib.axes.Axes (default: None) An existing matplotlib Axes. Creates one if ax=None. X_highlight : array-like, shape = [n_samples, n_features] (default: None) An array with data points that are used to highlight samples in X . zoom_factor : float (default: 1.0) Controls the scale of the x- and y-axis of the decision plot. hide_spines : bool (default: True) Hide axis spines if True. legend : int (default: 1) Integer to specify the legend location. No legend if legend is 0. markers : str (default: 's^oxv<>') Scatterplot markers. colors : str (default: 'red,blue,limegreen,gray,cyan') Comma separated list of colors. scatter_kwargs : dict (default: None) Keyword arguments for underlying matplotlib scatter function. contourf_kwargs : dict (default: None) Keyword arguments for underlying matplotlib contourf function. contour_kwargs : dict (default: None) Keyword arguments for underlying matplotlib contour function (which draws the lines between decision regions). scatter_highlight_kwargs : dict (default: None) Keyword arguments for underlying matplotlib scatter function. n_jobs : int or None, optional (default=None) The number of CPUs to use to do the computation using Python's multiprocessing library. None means 1. -1 means using all processors. New in v0.22.0. Returns ax : matplotlib.axes.Axes object Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/plotting/plot_decision_regions/","title":"plot_decision_regions"},{"location":"api_modules/mlxtend.plotting/plot_learning_curves/","text":"plot_learning_curves plot_learning_curves(X_train, y_train, X_test, y_test, clf, train_marker='o', test_marker='^', scoring='misclassification error', suppress_plot=False, print_model=True, title_fontsize=12, style='default', legend_loc='best') Plots learning curves of a classifier. Parameters X_train : array-like, shape = [n_samples, n_features] Feature matrix of the training dataset. y_train : array-like, shape = [n_samples] True class labels of the training dataset. X_test : array-like, shape = [n_samples, n_features] Feature matrix of the test dataset. y_test : array-like, shape = [n_samples] True class labels of the test dataset. clf : Classifier object. Must have a .predict .fit method. train_marker : str (default: 'o') Marker for the training set line plot. test_marker : str (default: '^') Marker for the test set line plot. scoring : str (default: 'misclassification error') If not 'misclassification error', accepts the following metrics (from scikit-learn): {'accuracy', 'average_precision', 'f1_micro', 'f1_macro', 'f1_weighted', 'f1_samples', 'log_loss', 'precision', 'recall', 'roc_auc', 'adjusted_rand_score', 'mean_absolute_error', 'mean_squared_error', 'median_absolute_error', 'r2'} suppress_plot=False : bool (default: False) Suppress matplotlib plots if True. Recommended for testing purposes. print_model : bool (default: True) Print model parameters in plot title if True. title_fontsize : int (default: 12) Determines the size of the plot title font. style : str (default: 'default') Matplotlib style. For more styles, please see https://matplotlib.org/stable/gallery/style_sheets/style_sheets_reference.html legend_loc : str (default: 'best') Where to place the plot legend: {'best', 'upper left', 'upper right', 'lower left', 'lower right'} Returns errors : (training_error, test_error): tuple of lists Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/plotting/plot_learning_curves/","title":"Plot learning curves"},{"location":"api_modules/mlxtend.plotting/plot_learning_curves/#plot_learning_curves","text":"plot_learning_curves(X_train, y_train, X_test, y_test, clf, train_marker='o', test_marker='^', scoring='misclassification error', suppress_plot=False, print_model=True, title_fontsize=12, style='default', legend_loc='best') Plots learning curves of a classifier. Parameters X_train : array-like, shape = [n_samples, n_features] Feature matrix of the training dataset. y_train : array-like, shape = [n_samples] True class labels of the training dataset. X_test : array-like, shape = [n_samples, n_features] Feature matrix of the test dataset. y_test : array-like, shape = [n_samples] True class labels of the test dataset. clf : Classifier object. Must have a .predict .fit method. train_marker : str (default: 'o') Marker for the training set line plot. test_marker : str (default: '^') Marker for the test set line plot. scoring : str (default: 'misclassification error') If not 'misclassification error', accepts the following metrics (from scikit-learn): {'accuracy', 'average_precision', 'f1_micro', 'f1_macro', 'f1_weighted', 'f1_samples', 'log_loss', 'precision', 'recall', 'roc_auc', 'adjusted_rand_score', 'mean_absolute_error', 'mean_squared_error', 'median_absolute_error', 'r2'} suppress_plot=False : bool (default: False) Suppress matplotlib plots if True. Recommended for testing purposes. print_model : bool (default: True) Print model parameters in plot title if True. title_fontsize : int (default: 12) Determines the size of the plot title font. style : str (default: 'default') Matplotlib style. For more styles, please see https://matplotlib.org/stable/gallery/style_sheets/style_sheets_reference.html legend_loc : str (default: 'best') Where to place the plot legend: {'best', 'upper left', 'upper right', 'lower left', 'lower right'} Returns errors : (training_error, test_error): tuple of lists Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/plotting/plot_learning_curves/","title":"plot_learning_curves"},{"location":"api_modules/mlxtend.plotting/plot_linear_regression/","text":"plot_linear_regression plot_linear_regression(X, y, model=LinearRegression(), corr_func='pearsonr', scattercolor='blue', fit_style='k--', legend=True, xlim='auto') Plot a linear regression line fit. Parameters X : numpy array, shape = [n_samples,] Samples. y : numpy array, shape (n_samples,) Target values model: object (default: sklearn.linear_model.LinearRegression) Estimator object for regression. Must implement a .fit() and .predict() method. corr_func: str or function (default: 'pearsonr') Uses pearsonr from scipy.stats if corr_func='pearsonr'. to compute the regression slope. If not 'pearsonr', the corr_func , the corr_func parameter expects a function of the form func( , ) as inputs, which is expected to return a tuple (<correlation_coefficient>, <some_unused_value>) . scattercolor: string (default: blue) Color of scatter plot points. fit_style: string (default: k--) Style for the line fit. legend: bool (default: True) Plots legend with corr_coeff coef., fit coef., and intercept values. xlim: array-like (x_min, x_max) or 'auto' (default: 'auto') X-axis limits for the linear line fit. Returns regression_fit : tuple intercept, slope, corr_coeff (float, float, float) Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/plotting/plot_linear_regression/","title":"Plot linear regression"},{"location":"api_modules/mlxtend.plotting/plot_linear_regression/#plot_linear_regression","text":"plot_linear_regression(X, y, model=LinearRegression(), corr_func='pearsonr', scattercolor='blue', fit_style='k--', legend=True, xlim='auto') Plot a linear regression line fit. Parameters X : numpy array, shape = [n_samples,] Samples. y : numpy array, shape (n_samples,) Target values model: object (default: sklearn.linear_model.LinearRegression) Estimator object for regression. Must implement a .fit() and .predict() method. corr_func: str or function (default: 'pearsonr') Uses pearsonr from scipy.stats if corr_func='pearsonr'. to compute the regression slope. If not 'pearsonr', the corr_func , the corr_func parameter expects a function of the form func( , ) as inputs, which is expected to return a tuple (<correlation_coefficient>, <some_unused_value>) . scattercolor: string (default: blue) Color of scatter plot points. fit_style: string (default: k--) Style for the line fit. legend: bool (default: True) Plots legend with corr_coeff coef., fit coef., and intercept values. xlim: array-like (x_min, x_max) or 'auto' (default: 'auto') X-axis limits for the linear line fit. Returns regression_fit : tuple intercept, slope, corr_coeff (float, float, float) Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/plotting/plot_linear_regression/","title":"plot_linear_regression"},{"location":"api_modules/mlxtend.plotting/plot_pca_correlation_graph/","text":"plot_pca_correlation_graph plot_pca_correlation_graph(X, variables_names, dimensions=(1, 2), figure_axis_size=6, X_pca=None, explained_variance=None) Compute the PCA for X and plots the Correlation graph Parameters X : 2d array like. The columns represent the different variables and the rows are the samples of thos variables variables_names : array like Name of the columns (the variables) of X dimensions: tuple with two elements. dimensions to be plotted (x,y) figure_axis_size : size of the final frame. The figure created is a square with length and width equal to figure_axis_size. X_pca : np.ndarray, shape = [n_samples, n_components]. Optional. X_pca is the matrix of the transformed components from X. If not provided, the function computes PCA automatically using mlxtend.feature_extraction.PrincipalComponentAnalysis Expected n_componentes >= max(dimensions) explained_variance : 1 dimension np.ndarray, length = n_components Optional. explained_variance are the eigenvalues from the diagonalized covariance matrix on the PCA transformatiopn. If not provided, the function computes PCA independently Expected n_componentes == X.shape[1] Returns matplotlib_figure, correlation_matrix Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/plotting/plot_pca_correlation_graph/","title":"Plot pca correlation graph"},{"location":"api_modules/mlxtend.plotting/plot_pca_correlation_graph/#plot_pca_correlation_graph","text":"plot_pca_correlation_graph(X, variables_names, dimensions=(1, 2), figure_axis_size=6, X_pca=None, explained_variance=None) Compute the PCA for X and plots the Correlation graph Parameters X : 2d array like. The columns represent the different variables and the rows are the samples of thos variables variables_names : array like Name of the columns (the variables) of X dimensions: tuple with two elements. dimensions to be plotted (x,y) figure_axis_size : size of the final frame. The figure created is a square with length and width equal to figure_axis_size. X_pca : np.ndarray, shape = [n_samples, n_components]. Optional. X_pca is the matrix of the transformed components from X. If not provided, the function computes PCA automatically using mlxtend.feature_extraction.PrincipalComponentAnalysis Expected n_componentes >= max(dimensions) explained_variance : 1 dimension np.ndarray, length = n_components Optional. explained_variance are the eigenvalues from the diagonalized covariance matrix on the PCA transformatiopn. If not provided, the function computes PCA independently Expected n_componentes == X.shape[1] Returns matplotlib_figure, correlation_matrix Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/plotting/plot_pca_correlation_graph/","title":"plot_pca_correlation_graph"},{"location":"api_modules/mlxtend.plotting/plot_sequential_feature_selection/","text":"plot_sequential_feature_selection plot_sequential_feature_selection(metric_dict, figsize=None, kind='std_dev', color='blue', bcolor='steelblue', marker='o', alpha=0.2, ylabel='Performance', confidence_interval=0.95) Plot feature selection results. Parameters metric_dict : mlxtend.SequentialFeatureSelector.get_metric_dict() object figsize : tuple (default: None) Height and width of the figure kind : str (default: \"std_dev\") The kind of error bar or confidence interval in {'std_dev', 'std_err', 'ci', None}. color : str (default: \"blue\") Color of the lineplot (accepts any matplotlib color name) bcolor : str (default: \"steelblue\"). Color of the error bars / confidence intervals (accepts any matplotlib color name). marker : str (default: \"o\") Marker of the line plot (accepts any matplotlib marker name). alpha : float in [0, 1] (default: 0.2) Transparency of the error bars / confidence intervals. ylabel : str (default: \"Performance\") Y-axis label. confidence_interval : float (default: 0.95) Confidence level if kind='ci' . Returns fig : matplotlib.pyplot.figure() object Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/plotting/plot_sequential_feature_selection/","title":"Plot sequential feature selection"},{"location":"api_modules/mlxtend.plotting/plot_sequential_feature_selection/#plot_sequential_feature_selection","text":"plot_sequential_feature_selection(metric_dict, figsize=None, kind='std_dev', color='blue', bcolor='steelblue', marker='o', alpha=0.2, ylabel='Performance', confidence_interval=0.95) Plot feature selection results. Parameters metric_dict : mlxtend.SequentialFeatureSelector.get_metric_dict() object figsize : tuple (default: None) Height and width of the figure kind : str (default: \"std_dev\") The kind of error bar or confidence interval in {'std_dev', 'std_err', 'ci', None}. color : str (default: \"blue\") Color of the lineplot (accepts any matplotlib color name) bcolor : str (default: \"steelblue\"). Color of the error bars / confidence intervals (accepts any matplotlib color name). marker : str (default: \"o\") Marker of the line plot (accepts any matplotlib marker name). alpha : float in [0, 1] (default: 0.2) Transparency of the error bars / confidence intervals. ylabel : str (default: \"Performance\") Y-axis label. confidence_interval : float (default: 0.95) Confidence level if kind='ci' . Returns fig : matplotlib.pyplot.figure() object Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/plotting/plot_sequential_feature_selection/","title":"plot_sequential_feature_selection"},{"location":"api_modules/mlxtend.plotting/remove_borders/","text":"remove_borders remove_borders(axes, left=False, bottom=False, right=True, top=True) Remove chart junk from matplotlib plots. Parameters axes : iterable An iterable containing plt.gca() or plt.subplot() objects, e.g. [plt.gca()]. left : bool (default: False ) Hide left axis spine if True. bottom : bool (default: False ) Hide bottom axis spine if True. right : bool (default: True ) Hide right axis spine if True. top : bool (default: True ) Hide top axis spine if True. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/plotting/remove_chartjunk/","title":"Remove borders"},{"location":"api_modules/mlxtend.plotting/remove_borders/#remove_borders","text":"remove_borders(axes, left=False, bottom=False, right=True, top=True) Remove chart junk from matplotlib plots. Parameters axes : iterable An iterable containing plt.gca() or plt.subplot() objects, e.g. [plt.gca()]. left : bool (default: False ) Hide left axis spine if True. bottom : bool (default: False ) Hide bottom axis spine if True. right : bool (default: True ) Hide right axis spine if True. top : bool (default: True ) Hide top axis spine if True. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/plotting/remove_chartjunk/","title":"remove_borders"},{"location":"api_modules/mlxtend.plotting/scatter_hist/","text":"scatter_hist scatter_hist(x, y, xlabel=None, ylabel=None, figsize=(5, 5)) Scatter plot and individual feature histograms along axes. Parameters x : 1D array-like or Pandas Series X-axis values. y : 1D array-like or Pandas Series Y-axis values. xlabel : str (default: None ) Label for the X-axis values. If x is a pandas Series, and xlabel is None , the label is inferred automatically. ylabel : str (default: None ) Label for the X-axis values. If y is a pandas Series, and ylabel is None , the label is inferred automatically. figsize : tuple (default: (5, 5) ) Matplotlib figure size. Returns plot : Matplotlib Figure object","title":"Scatter hist"},{"location":"api_modules/mlxtend.plotting/scatter_hist/#scatter_hist","text":"scatter_hist(x, y, xlabel=None, ylabel=None, figsize=(5, 5)) Scatter plot and individual feature histograms along axes. Parameters x : 1D array-like or Pandas Series X-axis values. y : 1D array-like or Pandas Series Y-axis values. xlabel : str (default: None ) Label for the X-axis values. If x is a pandas Series, and xlabel is None , the label is inferred automatically. ylabel : str (default: None ) Label for the X-axis values. If y is a pandas Series, and ylabel is None , the label is inferred automatically. figsize : tuple (default: (5, 5) ) Matplotlib figure size. Returns plot : Matplotlib Figure object","title":"scatter_hist"},{"location":"api_modules/mlxtend.plotting/scatterplotmatrix/","text":"scatterplotmatrix scatterplotmatrix(X, fig_axes=None, names=None, figsize=(8, 8), alpha=1.0, kwargs) Lower triangular of a scatterplot matrix Parameters X : array-like, shape={num_examples, num_features} Design matrix containing data instances (examples) with multiple exploratory variables (features). fix_axes : tuple (default: None) A (fig, axes) tuple, where fig is an figure object and axes is an axes object created via matplotlib, for example, by calling the pyplot subplot function fig, axes = plt.subplots(...) names : list (default: None) A list of string names, which should have the same number of elements as there are features (columns) in X . figsize : tuple (default: (8, 8)) Height and width of the subplot grid. Ignored if fig_axes is not None . alpha : float (default: 1.0) Transparency for both the scatter plots and the histograms along the diagonal. **kwargs : kwargs Keyword arguments for the scatterplots. Returns fix_axes : tuple A (fig, axes) tuple, where fig is an figure object and axes is an axes object created via matplotlib, for example, by calling the pyplot subplot function fig, axes = plt.subplots(...) Examples For more usage examples, please see https://rasbt.github.io/mlxtend/user_guide/plotting/scatterplotmatrix/","title":"Scatterplotmatrix"},{"location":"api_modules/mlxtend.plotting/scatterplotmatrix/#scatterplotmatrix","text":"scatterplotmatrix(X, fig_axes=None, names=None, figsize=(8, 8), alpha=1.0, kwargs) Lower triangular of a scatterplot matrix Parameters X : array-like, shape={num_examples, num_features} Design matrix containing data instances (examples) with multiple exploratory variables (features). fix_axes : tuple (default: None) A (fig, axes) tuple, where fig is an figure object and axes is an axes object created via matplotlib, for example, by calling the pyplot subplot function fig, axes = plt.subplots(...) names : list (default: None) A list of string names, which should have the same number of elements as there are features (columns) in X . figsize : tuple (default: (8, 8)) Height and width of the subplot grid. Ignored if fig_axes is not None . alpha : float (default: 1.0) Transparency for both the scatter plots and the histograms along the diagonal. **kwargs : kwargs Keyword arguments for the scatterplots. Returns fix_axes : tuple A (fig, axes) tuple, where fig is an figure object and axes is an axes object created via matplotlib, for example, by calling the pyplot subplot function fig, axes = plt.subplots(...) Examples For more usage examples, please see https://rasbt.github.io/mlxtend/user_guide/plotting/scatterplotmatrix/","title":"scatterplotmatrix"},{"location":"api_modules/mlxtend.plotting/stacked_barplot/","text":"stacked_barplot stacked_barplot(df, bar_width='auto', colors='bgrcky', labels='index', rotation=90, legend_loc='best') Function to plot stacked barplots Parameters df : pandas.DataFrame A pandas DataFrame where the index denotes the x-axis labels, and the columns contain the different measurements for each row. bar_width: 'auto' or float (default: 'auto') Parameter to set the widths of the bars. if 'auto', the width is automatically determined by the number of columns in the dataset. colors: str (default: 'bgrcky') The colors of the bars. labels: 'index' or iterable (default: 'index') If 'index', the DataFrame index will be used as x-tick labels. rotation: int (default: 90) Parameter to rotate the x-axis labels. legend_loc : str (default: 'best') Location of the plot legend {best, upper left, upper right, lower left, lower right} No legend if legend_loc=False Returns fig : matplotlib.pyplot figure object Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/plotting/stacked_barplot/","title":"Stacked barplot"},{"location":"api_modules/mlxtend.plotting/stacked_barplot/#stacked_barplot","text":"stacked_barplot(df, bar_width='auto', colors='bgrcky', labels='index', rotation=90, legend_loc='best') Function to plot stacked barplots Parameters df : pandas.DataFrame A pandas DataFrame where the index denotes the x-axis labels, and the columns contain the different measurements for each row. bar_width: 'auto' or float (default: 'auto') Parameter to set the widths of the bars. if 'auto', the width is automatically determined by the number of columns in the dataset. colors: str (default: 'bgrcky') The colors of the bars. labels: 'index' or iterable (default: 'index') If 'index', the DataFrame index will be used as x-tick labels. rotation: int (default: 90) Parameter to rotate the x-axis labels. legend_loc : str (default: 'best') Location of the plot legend {best, upper left, upper right, lower left, lower right} No legend if legend_loc=False Returns fig : matplotlib.pyplot figure object Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/plotting/stacked_barplot/","title":"stacked_barplot"},{"location":"api_modules/mlxtend.preprocessing/CopyTransformer/","text":"CopyTransformer CopyTransformer() Transformer that returns a copy of the input array For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/preprocessing/CopyTransformer/ Methods fit(X, y=None) Mock method. Does nothing. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] (default: None) Returns self fit_transform(X, y=None) Return a copy of the input array. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] (default: None) Returns X_copy : copy of the input X array. get_params(deep=True) Get parameters for this estimator. Parameters deep : bool, default=True If True, will return the parameters for this estimator and contained subobjects that are estimators. Returns params : dict Parameter names mapped to their values. set_params( params) Set the parameters of this estimator. The method works on simple estimators as well as on nested objects (such as :class:`~sklearn.pipeline.Pipeline`). The latter have parameters of the form ``<component>__<parameter>`` so that it's possible to update each component of a nested object. Parameters **params : dict Estimator parameters. Returns self : estimator instance Estimator instance. transform(X, y=None) Return a copy of the input array. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] (default: None) Returns X_copy : copy of the input X array.","title":"CopyTransformer"},{"location":"api_modules/mlxtend.preprocessing/CopyTransformer/#copytransformer","text":"CopyTransformer() Transformer that returns a copy of the input array For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/preprocessing/CopyTransformer/","title":"CopyTransformer"},{"location":"api_modules/mlxtend.preprocessing/CopyTransformer/#methods","text":"fit(X, y=None) Mock method. Does nothing. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] (default: None) Returns self fit_transform(X, y=None) Return a copy of the input array. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] (default: None) Returns X_copy : copy of the input X array. get_params(deep=True) Get parameters for this estimator. Parameters deep : bool, default=True If True, will return the parameters for this estimator and contained subobjects that are estimators. Returns params : dict Parameter names mapped to their values. set_params( params) Set the parameters of this estimator. The method works on simple estimators as well as on nested objects (such as :class:`~sklearn.pipeline.Pipeline`). The latter have parameters of the form ``<component>__<parameter>`` so that it's possible to update each component of a nested object. Parameters **params : dict Estimator parameters. Returns self : estimator instance Estimator instance. transform(X, y=None) Return a copy of the input array. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] (default: None) Returns X_copy : copy of the input X array.","title":"Methods"},{"location":"api_modules/mlxtend.preprocessing/DenseTransformer/","text":"DenseTransformer DenseTransformer(return_copy=True) Convert a sparse array into a dense array. For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/preprocessing/DenseTransformer/ Methods fit(X, y=None) Mock method. Does nothing. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] (default: None) Returns self fit_transform(X, y=None) Return a dense version of the input array. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] (default: None) Returns X_dense : dense version of the input X array. get_params(deep=True) Get parameters for this estimator. Parameters deep : bool, default=True If True, will return the parameters for this estimator and contained subobjects that are estimators. Returns params : dict Parameter names mapped to their values. set_params( params) Set the parameters of this estimator. The method works on simple estimators as well as on nested objects (such as :class:`~sklearn.pipeline.Pipeline`). The latter have parameters of the form ``<component>__<parameter>`` so that it's possible to update each component of a nested object. Parameters **params : dict Estimator parameters. Returns self : estimator instance Estimator instance. transform(X, y=None) Return a dense version of the input array. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] (default: None) Returns X_dense : dense version of the input X array.","title":"DenseTransformer"},{"location":"api_modules/mlxtend.preprocessing/DenseTransformer/#densetransformer","text":"DenseTransformer(return_copy=True) Convert a sparse array into a dense array. For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/preprocessing/DenseTransformer/","title":"DenseTransformer"},{"location":"api_modules/mlxtend.preprocessing/DenseTransformer/#methods","text":"fit(X, y=None) Mock method. Does nothing. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] (default: None) Returns self fit_transform(X, y=None) Return a dense version of the input array. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] (default: None) Returns X_dense : dense version of the input X array. get_params(deep=True) Get parameters for this estimator. Parameters deep : bool, default=True If True, will return the parameters for this estimator and contained subobjects that are estimators. Returns params : dict Parameter names mapped to their values. set_params( params) Set the parameters of this estimator. The method works on simple estimators as well as on nested objects (such as :class:`~sklearn.pipeline.Pipeline`). The latter have parameters of the form ``<component>__<parameter>`` so that it's possible to update each component of a nested object. Parameters **params : dict Estimator parameters. Returns self : estimator instance Estimator instance. transform(X, y=None) Return a dense version of the input array. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] (default: None) Returns X_dense : dense version of the input X array.","title":"Methods"},{"location":"api_modules/mlxtend.preprocessing/MeanCenterer/","text":"MeanCenterer MeanCenterer() Column centering of vectors and matrices. Attributes col_means : numpy.ndarray [n_columns] NumPy array storing the mean values for centering after fitting the MeanCenterer object. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/preprocessing/MeanCenterer/ Methods fit(X) Gets the column means for mean centering. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Array of data vectors, where n_samples is the number of samples and n_features is the number of features. Returns self fit_transform(X) Fits and transforms an arry. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Array of data vectors, where n_samples is the number of samples and n_features is the number of features. Returns X_tr : {array-like, sparse matrix}, shape = [n_samples, n_features] A copy of the input array with the columns centered. transform(X) Centers a NumPy array. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Array of data vectors, where n_samples is the number of samples and n_features is the number of features. Returns X_tr : {array-like, sparse matrix}, shape = [n_samples, n_features] A copy of the input array with the columns centered.","title":"MeanCenterer"},{"location":"api_modules/mlxtend.preprocessing/MeanCenterer/#meancenterer","text":"MeanCenterer() Column centering of vectors and matrices. Attributes col_means : numpy.ndarray [n_columns] NumPy array storing the mean values for centering after fitting the MeanCenterer object. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/preprocessing/MeanCenterer/","title":"MeanCenterer"},{"location":"api_modules/mlxtend.preprocessing/MeanCenterer/#methods","text":"fit(X) Gets the column means for mean centering. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Array of data vectors, where n_samples is the number of samples and n_features is the number of features. Returns self fit_transform(X) Fits and transforms an arry. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Array of data vectors, where n_samples is the number of samples and n_features is the number of features. Returns X_tr : {array-like, sparse matrix}, shape = [n_samples, n_features] A copy of the input array with the columns centered. transform(X) Centers a NumPy array. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Array of data vectors, where n_samples is the number of samples and n_features is the number of features. Returns X_tr : {array-like, sparse matrix}, shape = [n_samples, n_features] A copy of the input array with the columns centered.","title":"Methods"},{"location":"api_modules/mlxtend.preprocessing/TransactionEncoder/","text":"TransactionEncoder TransactionEncoder() Encoder class for transaction data in Python lists Parameters None Attributes columns_: list List of unique names in the X input list of lists Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/preprocessing/TransactionEncoder/ Methods fit(X) Learn unique column names from transaction DataFrame Parameters X : list of lists A python list of lists, where the outer list stores the n transactions and the inner list stores the items in each transaction. For example, [['Apple', 'Beer', 'Rice', 'Chicken'], ['Apple', 'Beer', 'Rice'], ['Apple', 'Beer'], ['Apple', 'Bananas'], ['Milk', 'Beer', 'Rice', 'Chicken'], ['Milk', 'Beer', 'Rice'], ['Milk', 'Beer'], ['Apple', 'Bananas']] fit_transform(X, sparse=False) Fit a TransactionEncoder encoder and transform a dataset. get_params(deep=True) Get parameters for this estimator. Parameters deep : bool, default=True If True, will return the parameters for this estimator and contained subobjects that are estimators. Returns params : dict Parameter names mapped to their values. inverse_transform(array) Transforms an encoded NumPy array back into transactions. Parameters array : NumPy array [n_transactions, n_unique_items] The NumPy one-hot encoded boolean array of the input transactions, where the columns represent the unique items found in the input array in alphabetic order For example, array([[True , False, True , True , False, True ], [True , False, True , False, False, True ], [True , False, True , False, False, False], [True , True , False, False, False, False], [False, False, True , True , True , True ], [False, False, True , False, True , True ], [False, False, True , False, True , False], [True , True , False, False, False, False]]) The corresponding column labels are available as self.columns_, e.g., ['Apple', 'Bananas', 'Beer', 'Chicken', 'Milk', 'Rice'] Returns X : list of lists A python list of lists, where the outer list stores the n transactions and the inner list stores the items in each transaction. For example, [['Apple', 'Beer', 'Rice', 'Chicken'], ['Apple', 'Beer', 'Rice'], ['Apple', 'Beer'], ['Apple', 'Bananas'], ['Milk', 'Beer', 'Rice', 'Chicken'], ['Milk', 'Beer', 'Rice'], ['Milk', 'Beer'], ['Apple', 'Bananas']] set_output( , transform=None)* Set output container. See :ref:`sphx_glr_auto_examples_miscellaneous_plot_set_output.py` for an example on how to use the API. Parameters transform : {\"default\", \"pandas\"}, default=None Configure output of transform and fit_transform . \"default\" : Default output format of a transformer \"pandas\" : DataFrame output None : Transform configuration is unchanged Returns self : estimator instance Estimator instance. set_params( params) Set the parameters of this estimator. The method works on simple estimators as well as on nested objects (such as :class:`~sklearn.pipeline.Pipeline`). The latter have parameters of the form ``<component>__<parameter>`` so that it's possible to update each component of a nested object. Parameters **params : dict Estimator parameters. Returns self : estimator instance Estimator instance. transform(X, sparse=False) Transform transactions into a one-hot encoded NumPy array. Parameters X : list of lists A python list of lists, where the outer list stores the n transactions and the inner list stores the items in each transaction. For example, [['Apple', 'Beer', 'Rice', 'Chicken'], ['Apple', 'Beer', 'Rice'], ['Apple', 'Beer'], ['Apple', 'Bananas'], ['Milk', 'Beer', 'Rice', 'Chicken'], ['Milk', 'Beer', 'Rice'], ['Milk', 'Beer'], ['Apple', 'Bananas']] sparse: bool (default=False) If True, transform will return Compressed Sparse Row matrix instead of the regular one. Returns array : NumPy array [n_transactions, n_unique_items] if sparse=False (default). Compressed Sparse Row matrix otherwise The one-hot encoded boolean array of the input transactions, where the columns represent the unique items found in the input array in alphabetic order. Exact representation depends on the sparse argument For example, array([[True , False, True , True , False, True ], [True , False, True , False, False, True ], [True , False, True , False, False, False], [True , True , False, False, False, False], [False, False, True , True , True , True ], [False, False, True , False, True , True ], [False, False, True , False, True , False], [True , True , False, False, False, False]]) The corresponding column labels are available as self.columns_, e.g., ['Apple', 'Bananas', 'Beer', 'Chicken', 'Milk', 'Rice']","title":"TransactionEncoder"},{"location":"api_modules/mlxtend.preprocessing/TransactionEncoder/#transactionencoder","text":"TransactionEncoder() Encoder class for transaction data in Python lists Parameters None Attributes columns_: list List of unique names in the X input list of lists Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/preprocessing/TransactionEncoder/","title":"TransactionEncoder"},{"location":"api_modules/mlxtend.preprocessing/TransactionEncoder/#methods","text":"fit(X) Learn unique column names from transaction DataFrame Parameters X : list of lists A python list of lists, where the outer list stores the n transactions and the inner list stores the items in each transaction. For example, [['Apple', 'Beer', 'Rice', 'Chicken'], ['Apple', 'Beer', 'Rice'], ['Apple', 'Beer'], ['Apple', 'Bananas'], ['Milk', 'Beer', 'Rice', 'Chicken'], ['Milk', 'Beer', 'Rice'], ['Milk', 'Beer'], ['Apple', 'Bananas']] fit_transform(X, sparse=False) Fit a TransactionEncoder encoder and transform a dataset. get_params(deep=True) Get parameters for this estimator. Parameters deep : bool, default=True If True, will return the parameters for this estimator and contained subobjects that are estimators. Returns params : dict Parameter names mapped to their values. inverse_transform(array) Transforms an encoded NumPy array back into transactions. Parameters array : NumPy array [n_transactions, n_unique_items] The NumPy one-hot encoded boolean array of the input transactions, where the columns represent the unique items found in the input array in alphabetic order For example, array([[True , False, True , True , False, True ], [True , False, True , False, False, True ], [True , False, True , False, False, False], [True , True , False, False, False, False], [False, False, True , True , True , True ], [False, False, True , False, True , True ], [False, False, True , False, True , False], [True , True , False, False, False, False]]) The corresponding column labels are available as self.columns_, e.g., ['Apple', 'Bananas', 'Beer', 'Chicken', 'Milk', 'Rice'] Returns X : list of lists A python list of lists, where the outer list stores the n transactions and the inner list stores the items in each transaction. For example, [['Apple', 'Beer', 'Rice', 'Chicken'], ['Apple', 'Beer', 'Rice'], ['Apple', 'Beer'], ['Apple', 'Bananas'], ['Milk', 'Beer', 'Rice', 'Chicken'], ['Milk', 'Beer', 'Rice'], ['Milk', 'Beer'], ['Apple', 'Bananas']] set_output( , transform=None)* Set output container. See :ref:`sphx_glr_auto_examples_miscellaneous_plot_set_output.py` for an example on how to use the API. Parameters transform : {\"default\", \"pandas\"}, default=None Configure output of transform and fit_transform . \"default\" : Default output format of a transformer \"pandas\" : DataFrame output None : Transform configuration is unchanged Returns self : estimator instance Estimator instance. set_params( params) Set the parameters of this estimator. The method works on simple estimators as well as on nested objects (such as :class:`~sklearn.pipeline.Pipeline`). The latter have parameters of the form ``<component>__<parameter>`` so that it's possible to update each component of a nested object. Parameters **params : dict Estimator parameters. Returns self : estimator instance Estimator instance. transform(X, sparse=False) Transform transactions into a one-hot encoded NumPy array. Parameters X : list of lists A python list of lists, where the outer list stores the n transactions and the inner list stores the items in each transaction. For example, [['Apple', 'Beer', 'Rice', 'Chicken'], ['Apple', 'Beer', 'Rice'], ['Apple', 'Beer'], ['Apple', 'Bananas'], ['Milk', 'Beer', 'Rice', 'Chicken'], ['Milk', 'Beer', 'Rice'], ['Milk', 'Beer'], ['Apple', 'Bananas']] sparse: bool (default=False) If True, transform will return Compressed Sparse Row matrix instead of the regular one. Returns array : NumPy array [n_transactions, n_unique_items] if sparse=False (default). Compressed Sparse Row matrix otherwise The one-hot encoded boolean array of the input transactions, where the columns represent the unique items found in the input array in alphabetic order. Exact representation depends on the sparse argument For example, array([[True , False, True , True , False, True ], [True , False, True , False, False, True ], [True , False, True , False, False, False], [True , True , False, False, False, False], [False, False, True , True , True , True ], [False, False, True , False, True , True ], [False, False, True , False, True , False], [True , True , False, False, False, False]]) The corresponding column labels are available as self.columns_, e.g., ['Apple', 'Bananas', 'Beer', 'Chicken', 'Milk', 'Rice']","title":"Methods"},{"location":"api_modules/mlxtend.preprocessing/minmax_scaling/","text":"minmax_scaling minmax_scaling(array, columns, min_val=0, max_val=1) Min max scaling of pandas' DataFrames. Parameters array : pandas DataFrame or NumPy ndarray, shape = [n_rows, n_columns]. columns : array-like, shape = [n_columns] Array-like with column names, e.g., ['col1', 'col2', ...] or column indices [0, 2, 4, ...] min_val : int or float , optional (default= 0 ) minimum value after rescaling. max_val : int or float , optional (default= 1 ) maximum value after rescaling. Returns df_new : pandas DataFrame object. Copy of the array or DataFrame with rescaled columns. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/preprocessing/minmax_scaling/","title":"Minmax scaling"},{"location":"api_modules/mlxtend.preprocessing/minmax_scaling/#minmax_scaling","text":"minmax_scaling(array, columns, min_val=0, max_val=1) Min max scaling of pandas' DataFrames. Parameters array : pandas DataFrame or NumPy ndarray, shape = [n_rows, n_columns]. columns : array-like, shape = [n_columns] Array-like with column names, e.g., ['col1', 'col2', ...] or column indices [0, 2, 4, ...] min_val : int or float , optional (default= 0 ) minimum value after rescaling. max_val : int or float , optional (default= 1 ) maximum value after rescaling. Returns df_new : pandas DataFrame object. Copy of the array or DataFrame with rescaled columns. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/preprocessing/minmax_scaling/","title":"minmax_scaling"},{"location":"api_modules/mlxtend.preprocessing/one_hot/","text":"one_hot one_hot(y, num_labels='auto', dtype='float') One-hot encoding of class labels Parameters y : array-like, shape = [n_classlabels] Python list or numpy array consisting of class labels. num_labels : int or 'auto' Number of unique labels in the class label array. Infers the number of unique labels from the input array if set to 'auto'. dtype : str NumPy array type (float, float32, float64) of the output array. Returns ary : numpy.ndarray, shape = [n_classlabels] One-hot encoded array, where each sample is represented as a row vector in the returned array. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/preprocessing/one_hot/","title":"One hot"},{"location":"api_modules/mlxtend.preprocessing/one_hot/#one_hot","text":"one_hot(y, num_labels='auto', dtype='float') One-hot encoding of class labels Parameters y : array-like, shape = [n_classlabels] Python list or numpy array consisting of class labels. num_labels : int or 'auto' Number of unique labels in the class label array. Infers the number of unique labels from the input array if set to 'auto'. dtype : str NumPy array type (float, float32, float64) of the output array. Returns ary : numpy.ndarray, shape = [n_classlabels] One-hot encoded array, where each sample is represented as a row vector in the returned array. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/preprocessing/one_hot/","title":"one_hot"},{"location":"api_modules/mlxtend.preprocessing/shuffle_arrays_unison/","text":"shuffle_arrays_unison shuffle_arrays_unison(arrays, random_seed=None) Shuffle NumPy arrays in unison. Parameters arrays : array-like, shape = [n_arrays] A list of NumPy arrays. random_seed : int (default: None) Sets the random state. Returns shuffled_arrays : A list of NumPy arrays after shuffling. Examples ``` >>> import numpy as np >>> from mlxtend.preprocessing import shuffle_arrays_unison >>> X1 = np.array([[1, 2, 3], [4, 5, 6], [7, 8, 9]]) >>> y1 = np.array([1, 2, 3]) >>> X2, y2 = shuffle_arrays_unison(arrays=[X1, y1], random_seed=3) >>> assert(X2.all() == np.array([[4, 5, 6], [1, 2, 3], [7, 8, 9]]).all()) >>> assert(y2.all() == np.array([2, 1, 3]).all()) >>> For more usage examples, please see https://rasbt.github.io/mlxtend/user_guide/preprocessing/shuffle_arrays_unison/ ```","title":"Shuffle arrays unison"},{"location":"api_modules/mlxtend.preprocessing/shuffle_arrays_unison/#shuffle_arrays_unison","text":"shuffle_arrays_unison(arrays, random_seed=None) Shuffle NumPy arrays in unison. Parameters arrays : array-like, shape = [n_arrays] A list of NumPy arrays. random_seed : int (default: None) Sets the random state. Returns shuffled_arrays : A list of NumPy arrays after shuffling. Examples ``` >>> import numpy as np >>> from mlxtend.preprocessing import shuffle_arrays_unison >>> X1 = np.array([[1, 2, 3], [4, 5, 6], [7, 8, 9]]) >>> y1 = np.array([1, 2, 3]) >>> X2, y2 = shuffle_arrays_unison(arrays=[X1, y1], random_seed=3) >>> assert(X2.all() == np.array([[4, 5, 6], [1, 2, 3], [7, 8, 9]]).all()) >>> assert(y2.all() == np.array([2, 1, 3]).all()) >>> For more usage examples, please see https://rasbt.github.io/mlxtend/user_guide/preprocessing/shuffle_arrays_unison/ ```","title":"shuffle_arrays_unison"},{"location":"api_modules/mlxtend.preprocessing/standardize/","text":"standardize standardize(array, columns=None, ddof=0, return_params=False, params=None) Standardize columns in pandas DataFrames. Parameters array : pandas DataFrame or NumPy ndarray, shape = [n_rows, n_columns]. columns : array-like, shape = [n_columns] (default: None) Array-like with column names, e.g., ['col1', 'col2', ...] or column indices [0, 2, 4, ...] If None, standardizes all columns. ddof : int (default: 0) Delta Degrees of Freedom. The divisor used in calculations is N - ddof, where N represents the number of elements. return_params : dict (default: False) If set to True, a dictionary is returned in addition to the standardized array. The parameter dictionary contains the column means ('avgs') and standard deviations ('stds') of the individual columns. params : dict (default: None) A dictionary with column means and standard deviations as returned by the standardize function if return_params was set to True. If a params dictionary is provided, the standardize function will use these instead of computing them from the current array. Notes If all values in a given column are the same, these values are all set to 0.0 . The standard deviation in the parameters dictionary is consequently set to 1.0 to avoid dividing by zero. Returns df_new : pandas DataFrame object. Copy of the array or DataFrame with standardized columns. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/preprocessing/standardize/","title":"Standardize"},{"location":"api_modules/mlxtend.preprocessing/standardize/#standardize","text":"standardize(array, columns=None, ddof=0, return_params=False, params=None) Standardize columns in pandas DataFrames. Parameters array : pandas DataFrame or NumPy ndarray, shape = [n_rows, n_columns]. columns : array-like, shape = [n_columns] (default: None) Array-like with column names, e.g., ['col1', 'col2', ...] or column indices [0, 2, 4, ...] If None, standardizes all columns. ddof : int (default: 0) Delta Degrees of Freedom. The divisor used in calculations is N - ddof, where N represents the number of elements. return_params : dict (default: False) If set to True, a dictionary is returned in addition to the standardized array. The parameter dictionary contains the column means ('avgs') and standard deviations ('stds') of the individual columns. params : dict (default: None) A dictionary with column means and standard deviations as returned by the standardize function if return_params was set to True. If a params dictionary is provided, the standardize function will use these instead of computing them from the current array. Notes If all values in a given column are the same, these values are all set to 0.0 . The standard deviation in the parameters dictionary is consequently set to 1.0 to avoid dividing by zero. Returns df_new : pandas DataFrame object. Copy of the array or DataFrame with standardized columns. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/preprocessing/standardize/","title":"standardize"},{"location":"api_modules/mlxtend.regressor/LinearRegression/","text":"LinearRegression LinearRegression(method='direct', eta=0.01, epochs=50, minibatches=None, random_seed=None, print_progress=0) Ordinary least squares linear regression. Parameters method : string (default: 'direct') For gradient descent-based optimization, use sgd (see minibatch parameter for further options). Otherwise, if direct (default), the analytical method is used. For alternative, numerically more stable solutions, use either qr (QR decomopisition) or svd (Singular Value Decomposition). eta : float (default: 0.01) solver learning rate (between 0.0 and 1.0). Used with method = 'sgd' . (See methods parameter for details) epochs : int (default: 50) Passes over the training dataset. Prior to each epoch, the dataset is shuffled if minibatches > 1 to prevent cycles in stochastic gradient descent. Used with method = 'sgd' . (See methods parameter for details) minibatches : int (default: None) The number of minibatches for gradient-based optimization. If None: Direct method, QR, or SVD method (see method parameter for details) If 1: Gradient Descent learning If len(y): Stochastic Gradient Descent learning If 1 < minibatches < len(y): Minibatch learning random_seed : int (default: None) Set random state for shuffling and initializing the weights. Used in method = 'sgd' . (See methods parameter for details) print_progress : int (default: 0) Prints progress in fitting to stderr if method = 'sgd' . 0: No output 1: Epochs elapsed and cost 2: 1 plus time elapsed 3: 2 plus estimated time until completion Attributes w_ : 2d-array, shape={n_features, 1} Model weights after fitting. b_ : 1d-array, shape={1,} Bias unit after fitting. cost_ : list Sum of squared errors after each epoch; ignored if solver='normal equation' Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/regressor/LinearRegression/ Methods fit(X, y, init_params=True) Learn model from training data. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values. init_params : bool (default: True) Re-initializes model parameters prior to fitting. Set False to continue training with weights from a previous model fitting. Returns self : object get_params(deep=True) Get parameters for this estimator. Parameters deep : boolean, optional If True, will return the parameters for this estimator and contained subobjects that are estimators. Returns params : mapping of string to any Parameter names mapped to their values.' adapted from https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/base.py Author: Gael Varoquaux gael.varoquaux@normalesup.org License: BSD 3 clause predict(X) Predict targets from X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns target_values : array-like, shape = [n_samples] Predicted target values. set_params( params) Set the parameters of this estimator. The method works on simple estimators as well as on nested objects (such as pipelines). The latter have parameters of the form <component>__<parameter> so that it's possible to update each component of a nested object. Returns self adapted from https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/base.py Author: Gael Varoquaux <gael.varoquaux@normalesup.org> License: BSD 3 clause","title":"LinearRegression"},{"location":"api_modules/mlxtend.regressor/LinearRegression/#linearregression","text":"LinearRegression(method='direct', eta=0.01, epochs=50, minibatches=None, random_seed=None, print_progress=0) Ordinary least squares linear regression. Parameters method : string (default: 'direct') For gradient descent-based optimization, use sgd (see minibatch parameter for further options). Otherwise, if direct (default), the analytical method is used. For alternative, numerically more stable solutions, use either qr (QR decomopisition) or svd (Singular Value Decomposition). eta : float (default: 0.01) solver learning rate (between 0.0 and 1.0). Used with method = 'sgd' . (See methods parameter for details) epochs : int (default: 50) Passes over the training dataset. Prior to each epoch, the dataset is shuffled if minibatches > 1 to prevent cycles in stochastic gradient descent. Used with method = 'sgd' . (See methods parameter for details) minibatches : int (default: None) The number of minibatches for gradient-based optimization. If None: Direct method, QR, or SVD method (see method parameter for details) If 1: Gradient Descent learning If len(y): Stochastic Gradient Descent learning If 1 < minibatches < len(y): Minibatch learning random_seed : int (default: None) Set random state for shuffling and initializing the weights. Used in method = 'sgd' . (See methods parameter for details) print_progress : int (default: 0) Prints progress in fitting to stderr if method = 'sgd' . 0: No output 1: Epochs elapsed and cost 2: 1 plus time elapsed 3: 2 plus estimated time until completion Attributes w_ : 2d-array, shape={n_features, 1} Model weights after fitting. b_ : 1d-array, shape={1,} Bias unit after fitting. cost_ : list Sum of squared errors after each epoch; ignored if solver='normal equation' Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/regressor/LinearRegression/","title":"LinearRegression"},{"location":"api_modules/mlxtend.regressor/LinearRegression/#methods","text":"fit(X, y, init_params=True) Learn model from training data. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values. init_params : bool (default: True) Re-initializes model parameters prior to fitting. Set False to continue training with weights from a previous model fitting. Returns self : object get_params(deep=True) Get parameters for this estimator. Parameters deep : boolean, optional If True, will return the parameters for this estimator and contained subobjects that are estimators. Returns params : mapping of string to any Parameter names mapped to their values.' adapted from https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/base.py Author: Gael Varoquaux gael.varoquaux@normalesup.org License: BSD 3 clause predict(X) Predict targets from X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns target_values : array-like, shape = [n_samples] Predicted target values. set_params( params) Set the parameters of this estimator. The method works on simple estimators as well as on nested objects (such as pipelines). The latter have parameters of the form <component>__<parameter> so that it's possible to update each component of a nested object. Returns self adapted from https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/base.py Author: Gael Varoquaux <gael.varoquaux@normalesup.org> License: BSD 3 clause","title":"Methods"},{"location":"api_modules/mlxtend.regressor/StackingCVRegressor/","text":"StackingCVRegressor StackingCVRegressor(regressors, meta_regressor, cv=5, shuffle=True, random_state=None, verbose=0, refit=True, use_features_in_secondary=False, store_train_meta_features=False, n_jobs=None, pre_dispatch='2 n_jobs', multi_output=False)* A 'Stacking Cross-Validation' regressor for scikit-learn estimators. Parameters regressors : array-like, shape = [n_regressors] A list of regressors. Invoking the fit method on the StackingCVRegressor will fit clones of these original regressors that will be stored in the class attribute self.regr_ . meta_regressor : object The meta-regressor to be fitted on the ensemble of regressor cv : int, cross-validation generator or iterable, optional (default: 5) Determines the cross-validation splitting strategy. Possible inputs for cv are: - None, to use the default 5-fold cross validation, - integer, to specify the number of folds in a KFold , - An object to be used as a cross-validation generator. - An iterable yielding train, test splits. For integer/None inputs, it will use KFold cross-validation shuffle : bool (default: True) If True, and the cv argument is integer, the training data will be shuffled at fitting stage prior to cross-validation. If the cv argument is a specific cross validation technique, this argument is omitted. random_state : int, RandomState instance or None, optional (default: None) Constrols the randomness of the cv splitter. Used when cv is integer and shuffle=True . New in v0.16.0. verbose : int, optional (default=0) Controls the verbosity of the building process. New in v0.16.0 refit : bool (default: True) Clones the regressors for stacking regression if True (default) or else uses the original ones, which will be refitted on the dataset upon calling the fit method. Setting refit=False is recommended if you are working with estimators that are supporting the scikit-learn fit/predict API interface but are not compatible to scikit-learn's clone function. use_features_in_secondary : bool (default: False) If True, the meta-regressor will be trained both on the predictions of the original regressors and the original dataset. If False, the meta-regressor will be trained only on the predictions of the original regressors. store_train_meta_features : bool (default: False) If True, the meta-features computed from the training data used for fitting the meta-regressor stored in the self.train_meta_features_ array, which can be accessed after calling fit . n_jobs : int or None, optional (default=None) The number of CPUs to use to do the computation. None means 1 unless in a :obj: joblib.parallel_backend context. -1 means using all processors. See :term: Glossary <n_jobs> for more details. New in v0.16.0. pre_dispatch : int, or string, optional Controls the number of jobs that get dispatched during parallel execution. Reducing this number can be useful to avoid an explosion of memory consumption when more jobs get dispatched than CPUs can process. This parameter can be: - None, in which case all the jobs are immediately created and spawned. Use this for lightweight and fast-running jobs, to avoid delays due to on-demand spawning of the jobs - An int, giving the exact number of total jobs that are spawned - A string, giving an expression as a function of n_jobs, as in '2*n_jobs' multi_output : bool (default: False) If True, allow multi-output targets, but forbid nan or inf values. If False, y will be checked to be a vector. (New in v0.19.0.) Attributes train_meta_features : numpy array, shape = [n_samples, n_regressors] meta-features for training data, where n_samples is the number of samples in training data and len(self.regressors) is the number of regressors. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/regressor/StackingCVRegressor/ Methods fit(X, y, groups=None, sample_weight=None) Fit ensemble regressors and the meta-regressor. Parameters X : numpy array, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : numpy array, shape = [n_samples] or [n_samples, n_targets] Target values. Multiple targets are supported only if self.multi_output is True. groups : numpy array/None, shape = [n_samples] The group that each sample belongs to. This is used by specific folding strategies such as GroupKFold() sample_weight : array-like, shape = [n_samples], optional Sample weights passed as sample_weights to each regressor in the regressors list as well as the meta_regressor. Raises error if some regressor does not support sample_weight in the fit() method. Returns self : object fit_transform(X, y=None, fit_params) Fit to data, then transform it. Fits transformer to `X` and `y` with optional parameters `fit_params` and returns a transformed version of `X`. Parameters X : array-like of shape (n_samples, n_features) Input samples. y : array-like of shape (n_samples,) or (n_samples, n_outputs), default=None Target values (None for unsupervised transformations). **fit_params : dict Additional fit parameters. Returns X_new : ndarray array of shape (n_samples, n_features_new) Transformed array. get_params(deep=True) Get parameters for this estimator. Parameters deep : bool, default=True If True, will return the parameters for this estimator and contained subobjects that are estimators. Returns params : dict Parameter names mapped to their values. predict(X) Predict target values for X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns y_target : array-like, shape = [n_samples] or [n_samples, n_targets] Predicted target values. predict_meta_features(X) Get meta-features of test-data. Parameters X : numpy array, shape = [n_samples, n_features] Test vectors, where n_samples is the number of samples and n_features is the number of features. Returns meta-features : numpy array, shape = [n_samples, len(self.regressors)] meta-features for test data, where n_samples is the number of samples in test data and len(self.regressors) is the number of regressors. If self.multi_output is True, then the number of columns is len(self.regressors) * n_targets. score(X, y, sample_weight=None) Return the coefficient of determination of the prediction. The coefficient of determination :math:`R^2` is defined as :math:`(1 - \\frac{u}{v})`, where :math:`u` is the residual sum of squares ((y_true - y_pred)** 2).sum() and :math: v is the total sum of squares ((y_true - y_true.mean()) ** 2).sum() . The best possible score is 1.0 and it can be negative (because the model can be arbitrarily worse). A constant model that always predicts the expected value of y , disregarding the input features, would get a :math: R^2 score of 0.0. Parameters X : array-like of shape (n_samples, n_features) Test samples. For some estimators this may be a precomputed kernel matrix or a list of generic objects instead with shape (n_samples, n_samples_fitted) , where n_samples_fitted is the number of samples used in the fitting for the estimator. y : array-like of shape (n_samples,) or (n_samples, n_outputs) True values for X . sample_weight : array-like of shape (n_samples,), default=None Sample weights. Returns score : float :math: R^2 of self.predict(X) w.r.t. y . Notes The :math: R^2 score used when calling score on a regressor uses multioutput='uniform_average' from version 0.23 to keep consistent with default value of :func: ~sklearn.metrics.r2_score . This influences the score method of all the multioutput regressors (except for :class: ~sklearn.multioutput.MultiOutputRegressor ). set_output( , transform=None)* Set output container. See :ref:`sphx_glr_auto_examples_miscellaneous_plot_set_output.py` for an example on how to use the API. Parameters transform : {\"default\", \"pandas\"}, default=None Configure output of transform and fit_transform . \"default\" : Default output format of a transformer \"pandas\" : DataFrame output None : Transform configuration is unchanged Returns self : estimator instance Estimator instance. set_params( params) Set the parameters of this estimator. Valid parameter keys can be listed with ``get_params()``. Returns self Properties named_regressors Returns List of named estimator tuples, like [('svc', SVC(...))]","title":"StackingCVRegressor"},{"location":"api_modules/mlxtend.regressor/StackingCVRegressor/#stackingcvregressor","text":"StackingCVRegressor(regressors, meta_regressor, cv=5, shuffle=True, random_state=None, verbose=0, refit=True, use_features_in_secondary=False, store_train_meta_features=False, n_jobs=None, pre_dispatch='2 n_jobs', multi_output=False)* A 'Stacking Cross-Validation' regressor for scikit-learn estimators. Parameters regressors : array-like, shape = [n_regressors] A list of regressors. Invoking the fit method on the StackingCVRegressor will fit clones of these original regressors that will be stored in the class attribute self.regr_ . meta_regressor : object The meta-regressor to be fitted on the ensemble of regressor cv : int, cross-validation generator or iterable, optional (default: 5) Determines the cross-validation splitting strategy. Possible inputs for cv are: - None, to use the default 5-fold cross validation, - integer, to specify the number of folds in a KFold , - An object to be used as a cross-validation generator. - An iterable yielding train, test splits. For integer/None inputs, it will use KFold cross-validation shuffle : bool (default: True) If True, and the cv argument is integer, the training data will be shuffled at fitting stage prior to cross-validation. If the cv argument is a specific cross validation technique, this argument is omitted. random_state : int, RandomState instance or None, optional (default: None) Constrols the randomness of the cv splitter. Used when cv is integer and shuffle=True . New in v0.16.0. verbose : int, optional (default=0) Controls the verbosity of the building process. New in v0.16.0 refit : bool (default: True) Clones the regressors for stacking regression if True (default) or else uses the original ones, which will be refitted on the dataset upon calling the fit method. Setting refit=False is recommended if you are working with estimators that are supporting the scikit-learn fit/predict API interface but are not compatible to scikit-learn's clone function. use_features_in_secondary : bool (default: False) If True, the meta-regressor will be trained both on the predictions of the original regressors and the original dataset. If False, the meta-regressor will be trained only on the predictions of the original regressors. store_train_meta_features : bool (default: False) If True, the meta-features computed from the training data used for fitting the meta-regressor stored in the self.train_meta_features_ array, which can be accessed after calling fit . n_jobs : int or None, optional (default=None) The number of CPUs to use to do the computation. None means 1 unless in a :obj: joblib.parallel_backend context. -1 means using all processors. See :term: Glossary <n_jobs> for more details. New in v0.16.0. pre_dispatch : int, or string, optional Controls the number of jobs that get dispatched during parallel execution. Reducing this number can be useful to avoid an explosion of memory consumption when more jobs get dispatched than CPUs can process. This parameter can be: - None, in which case all the jobs are immediately created and spawned. Use this for lightweight and fast-running jobs, to avoid delays due to on-demand spawning of the jobs - An int, giving the exact number of total jobs that are spawned - A string, giving an expression as a function of n_jobs, as in '2*n_jobs' multi_output : bool (default: False) If True, allow multi-output targets, but forbid nan or inf values. If False, y will be checked to be a vector. (New in v0.19.0.) Attributes train_meta_features : numpy array, shape = [n_samples, n_regressors] meta-features for training data, where n_samples is the number of samples in training data and len(self.regressors) is the number of regressors. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/regressor/StackingCVRegressor/","title":"StackingCVRegressor"},{"location":"api_modules/mlxtend.regressor/StackingCVRegressor/#methods","text":"fit(X, y, groups=None, sample_weight=None) Fit ensemble regressors and the meta-regressor. Parameters X : numpy array, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : numpy array, shape = [n_samples] or [n_samples, n_targets] Target values. Multiple targets are supported only if self.multi_output is True. groups : numpy array/None, shape = [n_samples] The group that each sample belongs to. This is used by specific folding strategies such as GroupKFold() sample_weight : array-like, shape = [n_samples], optional Sample weights passed as sample_weights to each regressor in the regressors list as well as the meta_regressor. Raises error if some regressor does not support sample_weight in the fit() method. Returns self : object fit_transform(X, y=None, fit_params) Fit to data, then transform it. Fits transformer to `X` and `y` with optional parameters `fit_params` and returns a transformed version of `X`. Parameters X : array-like of shape (n_samples, n_features) Input samples. y : array-like of shape (n_samples,) or (n_samples, n_outputs), default=None Target values (None for unsupervised transformations). **fit_params : dict Additional fit parameters. Returns X_new : ndarray array of shape (n_samples, n_features_new) Transformed array. get_params(deep=True) Get parameters for this estimator. Parameters deep : bool, default=True If True, will return the parameters for this estimator and contained subobjects that are estimators. Returns params : dict Parameter names mapped to their values. predict(X) Predict target values for X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns y_target : array-like, shape = [n_samples] or [n_samples, n_targets] Predicted target values. predict_meta_features(X) Get meta-features of test-data. Parameters X : numpy array, shape = [n_samples, n_features] Test vectors, where n_samples is the number of samples and n_features is the number of features. Returns meta-features : numpy array, shape = [n_samples, len(self.regressors)] meta-features for test data, where n_samples is the number of samples in test data and len(self.regressors) is the number of regressors. If self.multi_output is True, then the number of columns is len(self.regressors) * n_targets. score(X, y, sample_weight=None) Return the coefficient of determination of the prediction. The coefficient of determination :math:`R^2` is defined as :math:`(1 - \\frac{u}{v})`, where :math:`u` is the residual sum of squares ((y_true - y_pred)** 2).sum() and :math: v is the total sum of squares ((y_true - y_true.mean()) ** 2).sum() . The best possible score is 1.0 and it can be negative (because the model can be arbitrarily worse). A constant model that always predicts the expected value of y , disregarding the input features, would get a :math: R^2 score of 0.0. Parameters X : array-like of shape (n_samples, n_features) Test samples. For some estimators this may be a precomputed kernel matrix or a list of generic objects instead with shape (n_samples, n_samples_fitted) , where n_samples_fitted is the number of samples used in the fitting for the estimator. y : array-like of shape (n_samples,) or (n_samples, n_outputs) True values for X . sample_weight : array-like of shape (n_samples,), default=None Sample weights. Returns score : float :math: R^2 of self.predict(X) w.r.t. y . Notes The :math: R^2 score used when calling score on a regressor uses multioutput='uniform_average' from version 0.23 to keep consistent with default value of :func: ~sklearn.metrics.r2_score . This influences the score method of all the multioutput regressors (except for :class: ~sklearn.multioutput.MultiOutputRegressor ). set_output( , transform=None)* Set output container. See :ref:`sphx_glr_auto_examples_miscellaneous_plot_set_output.py` for an example on how to use the API. Parameters transform : {\"default\", \"pandas\"}, default=None Configure output of transform and fit_transform . \"default\" : Default output format of a transformer \"pandas\" : DataFrame output None : Transform configuration is unchanged Returns self : estimator instance Estimator instance. set_params( params) Set the parameters of this estimator. Valid parameter keys can be listed with ``get_params()``. Returns self","title":"Methods"},{"location":"api_modules/mlxtend.regressor/StackingCVRegressor/#properties","text":"named_regressors Returns List of named estimator tuples, like [('svc', SVC(...))]","title":"Properties"},{"location":"api_modules/mlxtend.regressor/StackingRegressor/","text":"StackingRegressor StackingRegressor(regressors, meta_regressor, verbose=0, use_features_in_secondary=False, store_train_meta_features=False, refit=True, multi_output=False) A Stacking regressor for scikit-learn estimators for regression. Parameters regressors : array-like, shape = [n_regressors] A list of regressors. Invoking the fit method on the StackingRegressor will fit clones of those original regressors that will be stored in the class attribute self.regr_ . meta_regressor : object The meta-regressor to be fitted on the ensemble of regressors verbose : int, optional (default=0) Controls the verbosity of the building process. - verbose=0 (default): Prints nothing - verbose=1 : Prints the number & name of the regressor being fitted - verbose=2 : Prints info about the parameters of the regressor being fitted - verbose>2 : Changes verbose param of the underlying regressor to self.verbose - 2 use_features_in_secondary : bool (default: False) If True, the meta-regressor will be trained both on the predictions of the original regressors and the original dataset. If False, the meta-regressor will be trained only on the predictions of the original regressors. store_train_meta_features : bool (default: False) If True, the meta-features computed from the training data used for fitting the meta-regressor stored in the self.train_meta_features_ array, which can be accessed after calling fit . Attributes regr_ : list, shape=[n_regressors] Fitted regressors (clones of the original regressors) meta_regr_ : estimator Fitted meta-regressor (clone of the original meta-estimator) coef_ : array-like, shape = [n_features] Model coefficients of the fitted meta-estimator intercept_ : float Intercept of the fitted meta-estimator train_meta_features : numpy array, shape = [n_samples, len(self.regressors)] meta-features for training data, where n_samples is the number of samples in training data and len(self.regressors) is the number of regressors. refit : bool (default: True) Clones the regressors for stacking regression if True (default) or else uses the original ones, which will be refitted on the dataset upon calling the fit method. Setting refit=False is recommended if you are working with estimators that are supporting the scikit-learn fit/predict API interface but are not compatible to scikit-learn's clone function. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/regressor/StackingRegressor/ Methods fit(X, y, sample_weight=None) Learn weight coefficients from training data for each regressor. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : numpy array, shape = [n_samples] or [n_samples, n_targets] Target values. Multiple targets are supported only if self.multi_output is True. sample_weight : array-like, shape = [n_samples], optional Sample weights passed as sample_weights to each regressor in the regressors list as well as the meta_regressor. Raises error if some regressor does not support sample_weight in the fit() method. Returns self : object fit_transform(X, y=None, fit_params) Fit to data, then transform it. Fits transformer to `X` and `y` with optional parameters `fit_params` and returns a transformed version of `X`. Parameters X : array-like of shape (n_samples, n_features) Input samples. y : array-like of shape (n_samples,) or (n_samples, n_outputs), default=None Target values (None for unsupervised transformations). **fit_params : dict Additional fit parameters. Returns X_new : ndarray array of shape (n_samples, n_features_new) Transformed array. get_params(deep=True) Return estimator parameter names for GridSearch support. predict(X) Predict target values for X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns y_target : array-like, shape = [n_samples] or [n_samples, n_targets] Predicted target values. predict_meta_features(X) Get meta-features of test-data. Parameters X : numpy array, shape = [n_samples, n_features] Test vectors, where n_samples is the number of samples and n_features is the number of features. Returns meta-features : numpy array, shape = [n_samples, len(self.regressors)] meta-features for test data, where n_samples is the number of samples in test data and len(self.regressors) is the number of regressors. If self.multi_output is True, then the number of columns is len(self.regressors) * n_targets score(X, y, sample_weight=None) Return the coefficient of determination of the prediction. The coefficient of determination :math:`R^2` is defined as :math:`(1 - \\frac{u}{v})`, where :math:`u` is the residual sum of squares ((y_true - y_pred)** 2).sum() and :math: v is the total sum of squares ((y_true - y_true.mean()) ** 2).sum() . The best possible score is 1.0 and it can be negative (because the model can be arbitrarily worse). A constant model that always predicts the expected value of y , disregarding the input features, would get a :math: R^2 score of 0.0. Parameters X : array-like of shape (n_samples, n_features) Test samples. For some estimators this may be a precomputed kernel matrix or a list of generic objects instead with shape (n_samples, n_samples_fitted) , where n_samples_fitted is the number of samples used in the fitting for the estimator. y : array-like of shape (n_samples,) or (n_samples, n_outputs) True values for X . sample_weight : array-like of shape (n_samples,), default=None Sample weights. Returns score : float :math: R^2 of self.predict(X) w.r.t. y . Notes The :math: R^2 score used when calling score on a regressor uses multioutput='uniform_average' from version 0.23 to keep consistent with default value of :func: ~sklearn.metrics.r2_score . This influences the score method of all the multioutput regressors (except for :class: ~sklearn.multioutput.MultiOutputRegressor ). set_output( , transform=None)* Set output container. See :ref:`sphx_glr_auto_examples_miscellaneous_plot_set_output.py` for an example on how to use the API. Parameters transform : {\"default\", \"pandas\"}, default=None Configure output of transform and fit_transform . \"default\" : Default output format of a transformer \"pandas\" : DataFrame output None : Transform configuration is unchanged Returns self : estimator instance Estimator instance. set_params( params) Set the parameters of this estimator. Valid parameter keys can be listed with ``get_params()``. Returns self Properties coef_ None intercept_ None named_regressors None","title":"StackingRegressor"},{"location":"api_modules/mlxtend.regressor/StackingRegressor/#stackingregressor","text":"StackingRegressor(regressors, meta_regressor, verbose=0, use_features_in_secondary=False, store_train_meta_features=False, refit=True, multi_output=False) A Stacking regressor for scikit-learn estimators for regression. Parameters regressors : array-like, shape = [n_regressors] A list of regressors. Invoking the fit method on the StackingRegressor will fit clones of those original regressors that will be stored in the class attribute self.regr_ . meta_regressor : object The meta-regressor to be fitted on the ensemble of regressors verbose : int, optional (default=0) Controls the verbosity of the building process. - verbose=0 (default): Prints nothing - verbose=1 : Prints the number & name of the regressor being fitted - verbose=2 : Prints info about the parameters of the regressor being fitted - verbose>2 : Changes verbose param of the underlying regressor to self.verbose - 2 use_features_in_secondary : bool (default: False) If True, the meta-regressor will be trained both on the predictions of the original regressors and the original dataset. If False, the meta-regressor will be trained only on the predictions of the original regressors. store_train_meta_features : bool (default: False) If True, the meta-features computed from the training data used for fitting the meta-regressor stored in the self.train_meta_features_ array, which can be accessed after calling fit . Attributes regr_ : list, shape=[n_regressors] Fitted regressors (clones of the original regressors) meta_regr_ : estimator Fitted meta-regressor (clone of the original meta-estimator) coef_ : array-like, shape = [n_features] Model coefficients of the fitted meta-estimator intercept_ : float Intercept of the fitted meta-estimator train_meta_features : numpy array, shape = [n_samples, len(self.regressors)] meta-features for training data, where n_samples is the number of samples in training data and len(self.regressors) is the number of regressors. refit : bool (default: True) Clones the regressors for stacking regression if True (default) or else uses the original ones, which will be refitted on the dataset upon calling the fit method. Setting refit=False is recommended if you are working with estimators that are supporting the scikit-learn fit/predict API interface but are not compatible to scikit-learn's clone function. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/regressor/StackingRegressor/","title":"StackingRegressor"},{"location":"api_modules/mlxtend.regressor/StackingRegressor/#methods","text":"fit(X, y, sample_weight=None) Learn weight coefficients from training data for each regressor. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : numpy array, shape = [n_samples] or [n_samples, n_targets] Target values. Multiple targets are supported only if self.multi_output is True. sample_weight : array-like, shape = [n_samples], optional Sample weights passed as sample_weights to each regressor in the regressors list as well as the meta_regressor. Raises error if some regressor does not support sample_weight in the fit() method. Returns self : object fit_transform(X, y=None, fit_params) Fit to data, then transform it. Fits transformer to `X` and `y` with optional parameters `fit_params` and returns a transformed version of `X`. Parameters X : array-like of shape (n_samples, n_features) Input samples. y : array-like of shape (n_samples,) or (n_samples, n_outputs), default=None Target values (None for unsupervised transformations). **fit_params : dict Additional fit parameters. Returns X_new : ndarray array of shape (n_samples, n_features_new) Transformed array. get_params(deep=True) Return estimator parameter names for GridSearch support. predict(X) Predict target values for X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns y_target : array-like, shape = [n_samples] or [n_samples, n_targets] Predicted target values. predict_meta_features(X) Get meta-features of test-data. Parameters X : numpy array, shape = [n_samples, n_features] Test vectors, where n_samples is the number of samples and n_features is the number of features. Returns meta-features : numpy array, shape = [n_samples, len(self.regressors)] meta-features for test data, where n_samples is the number of samples in test data and len(self.regressors) is the number of regressors. If self.multi_output is True, then the number of columns is len(self.regressors) * n_targets score(X, y, sample_weight=None) Return the coefficient of determination of the prediction. The coefficient of determination :math:`R^2` is defined as :math:`(1 - \\frac{u}{v})`, where :math:`u` is the residual sum of squares ((y_true - y_pred)** 2).sum() and :math: v is the total sum of squares ((y_true - y_true.mean()) ** 2).sum() . The best possible score is 1.0 and it can be negative (because the model can be arbitrarily worse). A constant model that always predicts the expected value of y , disregarding the input features, would get a :math: R^2 score of 0.0. Parameters X : array-like of shape (n_samples, n_features) Test samples. For some estimators this may be a precomputed kernel matrix or a list of generic objects instead with shape (n_samples, n_samples_fitted) , where n_samples_fitted is the number of samples used in the fitting for the estimator. y : array-like of shape (n_samples,) or (n_samples, n_outputs) True values for X . sample_weight : array-like of shape (n_samples,), default=None Sample weights. Returns score : float :math: R^2 of self.predict(X) w.r.t. y . Notes The :math: R^2 score used when calling score on a regressor uses multioutput='uniform_average' from version 0.23 to keep consistent with default value of :func: ~sklearn.metrics.r2_score . This influences the score method of all the multioutput regressors (except for :class: ~sklearn.multioutput.MultiOutputRegressor ). set_output( , transform=None)* Set output container. See :ref:`sphx_glr_auto_examples_miscellaneous_plot_set_output.py` for an example on how to use the API. Parameters transform : {\"default\", \"pandas\"}, default=None Configure output of transform and fit_transform . \"default\" : Default output format of a transformer \"pandas\" : DataFrame output None : Transform configuration is unchanged Returns self : estimator instance Estimator instance. set_params( params) Set the parameters of this estimator. Valid parameter keys can be listed with ``get_params()``. Returns self","title":"Methods"},{"location":"api_modules/mlxtend.regressor/StackingRegressor/#properties","text":"coef_ None intercept_ None named_regressors None","title":"Properties"},{"location":"api_modules/mlxtend.text/generalize_names/","text":"generalize_names generalize_names(name, output_sep=' ', firstname_output_letters=1) Generalize a person's first and last name. Returns a person's name in the format <last_name><separator><firstname letter(s)> (all lowercase) Parameters name : str Name of the player output_sep : str (default: ' ') String for separating last name and first name in the output. firstname_output_letters : int Number of letters in the abbreviated first name. Returns gen_name : str The generalized name. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/text/generalize_names/","title":"Generalize names"},{"location":"api_modules/mlxtend.text/generalize_names/#generalize_names","text":"generalize_names(name, output_sep=' ', firstname_output_letters=1) Generalize a person's first and last name. Returns a person's name in the format <last_name><separator><firstname letter(s)> (all lowercase) Parameters name : str Name of the player output_sep : str (default: ' ') String for separating last name and first name in the output. firstname_output_letters : int Number of letters in the abbreviated first name. Returns gen_name : str The generalized name. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/text/generalize_names/","title":"generalize_names"},{"location":"api_modules/mlxtend.text/generalize_names_duplcheck/","text":"generalize_names_duplcheck generalize_names_duplcheck(df, col_name) Generalizes names and removes duplicates. Applies mlxtend.text.generalize_names to a DataFrame with 1 first name letter by default and uses more first name letters if duplicates are detected. Parameters df : pandas.DataFrame DataFrame that contains a column where generalize_names should be applied. col_name : str Name of the DataFrame column where generalize_names function should be applied to. Returns df_new : str New DataFrame object where generalize_names function has been applied without duplicates. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/text/generalize_names_duplcheck/","title":"Generalize names duplcheck"},{"location":"api_modules/mlxtend.text/generalize_names_duplcheck/#generalize_names_duplcheck","text":"generalize_names_duplcheck(df, col_name) Generalizes names and removes duplicates. Applies mlxtend.text.generalize_names to a DataFrame with 1 first name letter by default and uses more first name letters if duplicates are detected. Parameters df : pandas.DataFrame DataFrame that contains a column where generalize_names should be applied. col_name : str Name of the DataFrame column where generalize_names function should be applied to. Returns df_new : str New DataFrame object where generalize_names function has been applied without duplicates. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/text/generalize_names_duplcheck/","title":"generalize_names_duplcheck"},{"location":"api_modules/mlxtend.text/tokenizer_emoticons/","text":"tokenizer_emoticons tokenizer_emoticons(text) Return emoticons from text Examples >>> tokenizer_emoticons('</a>This :) is :( a test :-)!') [':)', ':(', ':-)'] For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/text/tokenizer_emoticons/","title":"Tokenizer emoticons"},{"location":"api_modules/mlxtend.text/tokenizer_emoticons/#tokenizer_emoticons","text":"tokenizer_emoticons(text) Return emoticons from text Examples >>> tokenizer_emoticons('</a>This :) is :( a test :-)!') [':)', ':(', ':-)'] For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/text/tokenizer_emoticons/","title":"tokenizer_emoticons"},{"location":"api_modules/mlxtend.text/tokenizer_words_and_emoticons/","text":"tokenizer_words_and_emoticons tokenizer_words_and_emoticons(text) Convert text to lowercase words and emoticons. Examples >>> tokenizer_words_and_emoticons('</a>This :) is :( a test :-)!') ['this', 'is', 'a', 'test', ':)', ':(', ':-)'] For more usage examples, please see https://rasbt.github.io/mlxtend/user_guide/text/tokenizer_words_and_emoticons/","title":"Tokenizer words and emoticons"},{"location":"api_modules/mlxtend.text/tokenizer_words_and_emoticons/#tokenizer_words_and_emoticons","text":"tokenizer_words_and_emoticons(text) Convert text to lowercase words and emoticons. Examples >>> tokenizer_words_and_emoticons('</a>This :) is :( a test :-)!') ['this', 'is', 'a', 'test', ':)', ':(', ':-)'] For more usage examples, please see https://rasbt.github.io/mlxtend/user_guide/text/tokenizer_words_and_emoticons/","title":"tokenizer_words_and_emoticons"},{"location":"api_modules/mlxtend.utils/Counter/","text":"Counter Counter(stderr=False, start_newline=True, precision=0, name=None) Class to display the progress of for-loop iterators. Parameters stderr : bool (default: True) Prints output to sys.stderr if True; uses sys.stdout otherwise. start_newline : bool (default: True) Prepends a new line to the counter, which prevents overwriting counters if multiple counters are printed in succession. precision: int (default: 0) Sets the number of decimal places when displaying the time elapsed in seconds. name : string (default: None) Prepends the specified name before the counter to allow distinguishing between multiple counters. Attributes curr_iter : int The current iteration. start_time : float The system's time in seconds when the Counter was initialized. end_time : float The system's time in seconds when the Counter was last updated. Examples >>> cnt = Counter() >>> for i in range(20): ... # do some computation ... time.sleep(0.1) ... cnt.update() 20 iter | 2 sec >>> print('The counter was initialized.' ' %d seconds ago.' % (time.time() - cnt.start_time)) The counter was initialized 2 seconds ago >>> print('The counter was last updated' ' %d seconds ago.' % (time.time() - cnt.end_time)) The counter was last updated 0 seconds ago. For more usage examples, please see https://rasbt.github.io/mlxtend/user_guide/utils/Counter/ Methods update() Print current iteration and time elapsed.","title":"Counter"},{"location":"api_modules/mlxtend.utils/Counter/#counter","text":"Counter(stderr=False, start_newline=True, precision=0, name=None) Class to display the progress of for-loop iterators. Parameters stderr : bool (default: True) Prints output to sys.stderr if True; uses sys.stdout otherwise. start_newline : bool (default: True) Prepends a new line to the counter, which prevents overwriting counters if multiple counters are printed in succession. precision: int (default: 0) Sets the number of decimal places when displaying the time elapsed in seconds. name : string (default: None) Prepends the specified name before the counter to allow distinguishing between multiple counters. Attributes curr_iter : int The current iteration. start_time : float The system's time in seconds when the Counter was initialized. end_time : float The system's time in seconds when the Counter was last updated. Examples >>> cnt = Counter() >>> for i in range(20): ... # do some computation ... time.sleep(0.1) ... cnt.update() 20 iter | 2 sec >>> print('The counter was initialized.' ' %d seconds ago.' % (time.time() - cnt.start_time)) The counter was initialized 2 seconds ago >>> print('The counter was last updated' ' %d seconds ago.' % (time.time() - cnt.end_time)) The counter was last updated 0 seconds ago. For more usage examples, please see https://rasbt.github.io/mlxtend/user_guide/utils/Counter/","title":"Counter"},{"location":"api_modules/mlxtend.utils/Counter/#methods","text":"update() Print current iteration and time elapsed.","title":"Methods"},{"location":"api_modules/mlxtend.utils/assert_raises/","text":"assert_raises assert_raises(exception_type, message, func, args, * kwargs) Check that an exception is raised with a specific message Parameters exception_type : exception The exception that should be raised message : str (default: None) The error message that should be raised. Ignored if False or None. func : callable The function that raises the exception *args : positional arguments to func . **kwargs : keyword arguments to func","title":"Assert raises"},{"location":"api_modules/mlxtend.utils/assert_raises/#assert_raises","text":"assert_raises(exception_type, message, func, args, * kwargs) Check that an exception is raised with a specific message Parameters exception_type : exception The exception that should be raised message : str (default: None) The error message that should be raised. Ignored if False or None. func : callable The function that raises the exception *args : positional arguments to func . **kwargs : keyword arguments to func","title":"assert_raises"},{"location":"api_modules/mlxtend.utils/check_Xy/","text":"check_Xy check_Xy(X, y, y_int=True) None","title":"check Xy"},{"location":"api_modules/mlxtend.utils/check_Xy/#check_xy","text":"check_Xy(X, y, y_int=True) None","title":"check_Xy"},{"location":"api_modules/mlxtend.utils/format_kwarg_dictionaries/","text":"format_kwarg_dictionaries format_kwarg_dictionaries(default_kwargs=None, user_kwargs=None, protected_keys=None) Function to combine default and user specified kwargs dictionaries Parameters default_kwargs : dict, optional Default kwargs (default is None). user_kwargs : dict, optional User specified kwargs (default is None). protected_keys : array_like, optional Sequence of keys to be removed from the returned dictionary (default is None). Returns formatted_kwargs : dict Formatted kwargs dictionary.","title":"Format kwarg dictionaries"},{"location":"api_modules/mlxtend.utils/format_kwarg_dictionaries/#format_kwarg_dictionaries","text":"format_kwarg_dictionaries(default_kwargs=None, user_kwargs=None, protected_keys=None) Function to combine default and user specified kwargs dictionaries Parameters default_kwargs : dict, optional Default kwargs (default is None). user_kwargs : dict, optional User specified kwargs (default is None). protected_keys : array_like, optional Sequence of keys to be removed from the returned dictionary (default is None). Returns formatted_kwargs : dict Formatted kwargs dictionary.","title":"format_kwarg_dictionaries"},{"location":"api_subpackages/mlxtend._base/","text":"mlxtend version: 0.23.1","title":"Mlxtend. base"},{"location":"api_subpackages/mlxtend.classifier/","text":"mlxtend version: 0.23.1 Adaline Adaline(eta=0.01, epochs=50, minibatches=None, random_seed=None, print_progress=0) ADAptive LInear NEuron classifier. Note that this implementation of Adaline expects binary class labels in {0, 1}. Parameters eta : float (default: 0.01) solver rate (between 0.0 and 1.0) epochs : int (default: 50) Passes over the training dataset. Prior to each epoch, the dataset is shuffled if minibatches > 1 to prevent cycles in stochastic gradient descent. minibatches : int (default: None) The number of minibatches for gradient-based optimization. If None: Normal Equations (closed-form solution) If 1: Gradient Descent learning If len(y): Stochastic Gradient Descent (SGD) online learning If 1 < minibatches < len(y): SGD Minibatch learning random_seed : int (default: None) Set random state for shuffling and initializing the weights. print_progress : int (default: 0) Prints progress in fitting to stderr if not solver='normal equation' 0: No output 1: Epochs elapsed and cost 2: 1 plus time elapsed 3: 2 plus estimated time until completion Attributes w_ : 2d-array, shape={n_features, 1} Model weights after fitting. b_ : 1d-array, shape={1,} Bias unit after fitting. cost_ : list Sum of squared errors after each epoch. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/classifier/Adaline/ Methods fit(X, y, init_params=True) Learn model from training data. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values. init_params : bool (default: True) Re-initializes model parameters prior to fitting. Set False to continue training with weights from a previous model fitting. Returns self : object get_params(deep=True) Get parameters for this estimator. Parameters deep : boolean, optional If True, will return the parameters for this estimator and contained subobjects that are estimators. Returns params : mapping of string to any Parameter names mapped to their values.' adapted from https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/base.py Author: Gael Varoquaux gael.varoquaux@normalesup.org License: BSD 3 clause predict(X) Predict targets from X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns target_values : array-like, shape = [n_samples] Predicted target values. score(X, y) Compute the prediction accuracy Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values (true class labels). Returns acc : float The prediction accuracy as a float between 0.0 and 1.0 (perfect score). set_params( params) Set the parameters of this estimator. The method works on simple estimators as well as on nested objects (such as pipelines). The latter have parameters of the form <component>__<parameter> so that it's possible to update each component of a nested object. Returns self adapted from https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/base.py Author: Gael Varoquaux <gael.varoquaux@normalesup.org> License: BSD 3 clause EnsembleVoteClassifier EnsembleVoteClassifier(clfs, voting='hard', weights=None, verbose=0, use_clones=True, fit_base_estimators=True) Soft Voting/Majority Rule classifier for scikit-learn estimators. Parameters clfs : array-like, shape = [n_classifiers] A list of classifiers. Invoking the fit method on the VotingClassifier will fit clones of those original classifiers be stored in the class attribute if use_clones=True (default) and fit_base_estimators=True (default). voting : str, {'hard', 'soft'} (default='hard') If 'hard', uses predicted class labels for majority rule voting. Else if 'soft', predicts the class label based on the argmax of the sums of the predicted probalities, which is recommended for an ensemble of well-calibrated classifiers. weights : array-like, shape = [n_classifiers], optional (default= None ) Sequence of weights ( float or int ) to weight the occurances of predicted class labels ( hard voting) or class probabilities before averaging ( soft voting). Uses uniform weights if None . verbose : int, optional (default=0) Controls the verbosity of the building process. - verbose=0 (default): Prints nothing - verbose=1 : Prints the number & name of the clf being fitted - verbose=2 : Prints info about the parameters of the clf being fitted - verbose>2 : Changes verbose param of the underlying clf to self.verbose - 2 use_clones : bool (default: True) Clones the classifiers for stacking classification if True (default) or else uses the original ones, which will be refitted on the dataset upon calling the fit method. Hence, if use_clones=True, the original input classifiers will remain unmodified upon using the StackingClassifier's fit method. Setting use_clones=False is recommended if you are working with estimators that are supporting the scikit-learn fit/predict API interface but are not compatible to scikit-learn's clone function. fit_base_estimators : bool (default: True) Refits classifiers in clfs if True; uses references to the clfs , otherwise (assumes that the classifiers were already fit). Note: fit_base_estimators=False will enforce use_clones to be False, and is incompatible to most scikit-learn wrappers! For instance, if any form of cross-validation is performed this would require the re-fitting classifiers to training folds, which would raise a NotFitterError if fit_base_estimators=False. (New in mlxtend v0.6.) Attributes classes_ : array-like, shape = [n_predictions] clf : array-like, shape = [n_predictions] The input classifiers; may be overwritten if use_clones=False clf_ : array-like, shape = [n_predictions] Fitted input classifiers; clones if use_clones=True Examples ``` >>> import numpy as np >>> from sklearn.linear_model import LogisticRegression >>> from sklearn.naive_bayes import GaussianNB >>> from sklearn.ensemble import RandomForestClassifier >>> from mlxtend.sklearn import EnsembleVoteClassifier >>> clf1 = LogisticRegression(random_seed=1) >>> clf2 = RandomForestClassifier(random_seed=1) >>> clf3 = GaussianNB() >>> X = np.array([[-1, -1], [-2, -1], [-3, -2], [1, 1], [2, 1], [3, 2]]) >>> y = np.array([1, 1, 1, 2, 2, 2]) >>> eclf1 = EnsembleVoteClassifier(clfs=[clf1, clf2, clf3], ... voting='hard', verbose=1) >>> eclf1 = eclf1.fit(X, y) >>> print(eclf1.predict(X)) [1 1 1 2 2 2] >>> eclf2 = EnsembleVoteClassifier(clfs=[clf1, clf2, clf3], voting='soft') >>> eclf2 = eclf2.fit(X, y) >>> print(eclf2.predict(X)) [1 1 1 2 2 2] >>> eclf3 = EnsembleVoteClassifier(clfs=[clf1, clf2, clf3], ... voting='soft', weights=[2,1,1]) >>> eclf3 = eclf3.fit(X, y) >>> print(eclf3.predict(X)) [1 1 1 2 2 2] >>> For more usage examples, please see https://rasbt.github.io/mlxtend/user_guide/classifier/EnsembleVoteClassifier/ ``` Methods fit(X, y, sample_weight=None) Learn weight coefficients from training data for each classifier. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values. sample_weight : array-like, shape = [n_samples], optional Sample weights passed as sample_weights to each regressor in the regressors list as well as the meta_regressor. Raises error if some regressor does not support sample_weight in the fit() method. Returns self : object fit_transform(X, y=None, fit_params) Fit to data, then transform it. Fits transformer to `X` and `y` with optional parameters `fit_params` and returns a transformed version of `X`. Parameters X : array-like of shape (n_samples, n_features) Input samples. y : array-like of shape (n_samples,) or (n_samples, n_outputs), default=None Target values (None for unsupervised transformations). **fit_params : dict Additional fit parameters. Returns X_new : ndarray array of shape (n_samples, n_features_new) Transformed array. get_params(deep=True) Return estimator parameter names for GridSearch support. predict(X) Predict class labels for X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns maj : array-like, shape = [n_samples] Predicted class labels. predict_proba(X) Predict class probabilities for X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns avg : array-like, shape = [n_samples, n_classes] Weighted average probability for each class per sample. score(X, y, sample_weight=None) Return the mean accuracy on the given test data and labels. In multi-label classification, this is the subset accuracy which is a harsh metric since you require for each sample that each label set be correctly predicted. Parameters X : array-like of shape (n_samples, n_features) Test samples. y : array-like of shape (n_samples,) or (n_samples, n_outputs) True labels for X . sample_weight : array-like of shape (n_samples,), default=None Sample weights. Returns score : float Mean accuracy of self.predict(X) w.r.t. y . set_output( , transform=None)* Set output container. See :ref:`sphx_glr_auto_examples_miscellaneous_plot_set_output.py` for an example on how to use the API. Parameters transform : {\"default\", \"pandas\"}, default=None Configure output of transform and fit_transform . \"default\" : Default output format of a transformer \"pandas\" : DataFrame output None : Transform configuration is unchanged Returns self : estimator instance Estimator instance. set_params( params) Set the parameters of this estimator. The method works on simple estimators as well as on nested objects (such as :class:`~sklearn.pipeline.Pipeline`). The latter have parameters of the form ``<component>__<parameter>`` so that it's possible to update each component of a nested object. Parameters **params : dict Estimator parameters. Returns self : estimator instance Estimator instance. transform(X) Return class labels or probabilities for X for each estimator. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns If voting='soft'`` : array-like = [n_classifiers, n_samples, n_classes] Class probabilties calculated by each classifier. If voting='hard'`` : array-like = [n_classifiers, n_samples] Class labels predicted by each classifier. LogisticRegression LogisticRegression(eta=0.01, epochs=50, l2_lambda=0.0, minibatches=1, random_seed=None, print_progress=0) Logistic regression classifier. Note that this implementation of Logistic Regression expects binary class labels in {0, 1}. Parameters eta : float (default: 0.01) Learning rate (between 0.0 and 1.0) epochs : int (default: 50) Passes over the training dataset. Prior to each epoch, the dataset is shuffled if minibatches > 1 to prevent cycles in stochastic gradient descent. l2_lambda : float Regularization parameter for L2 regularization. No regularization if l2_lambda=0.0. minibatches : int (default: 1) The number of minibatches for gradient-based optimization. If 1: Gradient Descent learning If len(y): Stochastic Gradient Descent (SGD) online learning If 1 < minibatches < len(y): SGD Minibatch learning random_seed : int (default: None) Set random state for shuffling and initializing the weights. print_progress : int (default: 0) Prints progress in fitting to stderr. 0: No output 1: Epochs elapsed and cost 2: 1 plus time elapsed 3: 2 plus estimated time until completion Attributes w_ : 2d-array, shape={n_features, 1} Model weights after fitting. b_ : 1d-array, shape={1,} Bias unit after fitting. cost_ : list List of floats with cross_entropy cost (sgd or gd) for every epoch. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/classifier/LogisticRegression/ Methods fit(X, y, init_params=True) Learn model from training data. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values. init_params : bool (default: True) Re-initializes model parameters prior to fitting. Set False to continue training with weights from a previous model fitting. Returns self : object get_params(deep=True) Get parameters for this estimator. Parameters deep : boolean, optional If True, will return the parameters for this estimator and contained subobjects that are estimators. Returns params : mapping of string to any Parameter names mapped to their values.' adapted from https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/base.py Author: Gael Varoquaux gael.varoquaux@normalesup.org License: BSD 3 clause predict(X) Predict targets from X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns target_values : array-like, shape = [n_samples] Predicted target values. predict_proba(X) Predict class probabilities of X from the net input. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns Class 1 probability : float score(X, y) Compute the prediction accuracy Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values (true class labels). Returns acc : float The prediction accuracy as a float between 0.0 and 1.0 (perfect score). set_params( params) Set the parameters of this estimator. The method works on simple estimators as well as on nested objects (such as pipelines). The latter have parameters of the form <component>__<parameter> so that it's possible to update each component of a nested object. Returns self adapted from https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/base.py Author: Gael Varoquaux <gael.varoquaux@normalesup.org> License: BSD 3 clause MultiLayerPerceptron MultiLayerPerceptron(eta=0.5, epochs=50, hidden_layers=[50], n_classes=None, momentum=0.0, l1=0.0, l2=0.0, dropout=1.0, decrease_const=0.0, minibatches=1, random_seed=None, print_progress=0) Multi-layer perceptron classifier with logistic sigmoid activations Parameters eta : float (default: 0.5) Learning rate (between 0.0 and 1.0) epochs : int (default: 50) Passes over the training dataset. Prior to each epoch, the dataset is shuffled if minibatches > 1 to prevent cycles in stochastic gradient descent. hidden_layers : list (default: [50]) Number of units per hidden layer. By default 50 units in the first hidden layer. At the moment only 1 hidden layer is supported n_classes : int (default: None) A positive integer to declare the number of class labels if not all class labels are present in a partial training set. Gets the number of class labels automatically if None. l1 : float (default: 0.0) L1 regularization strength l2 : float (default: 0.0) L2 regularization strength momentum : float (default: 0.0) Momentum constant. Factor multiplied with the gradient of the previous epoch t-1 to improve learning speed w(t) := w(t) - (grad(t) + momentum * grad(t-1)) decrease_const : float (default: 0.0) Decrease constant. Shrinks the learning rate after each epoch via eta / (1 + epoch*decrease_const) minibatches : int (default: 1) Divide the training data into k minibatches for accelerated stochastic gradient descent learning. Gradient Descent Learning if minibatches = 1 Stochastic Gradient Descent learning if minibatches = len(y) Minibatch learning if minibatches > 1 random_seed : int (default: None) Set random state for shuffling and initializing the weights. print_progress : int (default: 0) Prints progress in fitting to stderr. 0: No output 1: Epochs elapsed and cost 2: 1 plus time elapsed 3: 2 plus estimated time until completion Attributes w_ : 2d-array, shape=[n_features, n_classes] Weights after fitting. b_ : 1D-array, shape=[n_classes] Bias units after fitting. cost_ : list List of floats; the mean categorical cross entropy cost after each epoch. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/classifier/MultiLayerPerceptron/ Methods fit(X, y, init_params=True) Learn model from training data. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values. init_params : bool (default: True) Re-initializes model parameters prior to fitting. Set False to continue training with weights from a previous model fitting. Returns self : object get_params(deep=True) Get parameters for this estimator. Parameters deep : boolean, optional If True, will return the parameters for this estimator and contained subobjects that are estimators. Returns params : mapping of string to any Parameter names mapped to their values.' adapted from https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/base.py Author: Gael Varoquaux gael.varoquaux@normalesup.org License: BSD 3 clause predict(X) Predict targets from X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns target_values : array-like, shape = [n_samples] Predicted target values. predict_proba(X) Predict class probabilities of X from the net input. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns Class probabilties : array-like, shape= [n_samples, n_classes] score(X, y) Compute the prediction accuracy Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values (true class labels). Returns acc : float The prediction accuracy as a float between 0.0 and 1.0 (perfect score). set_params( params) Set the parameters of this estimator. The method works on simple estimators as well as on nested objects (such as pipelines). The latter have parameters of the form <component>__<parameter> so that it's possible to update each component of a nested object. Returns self adapted from https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/base.py Author: Gael Varoquaux <gael.varoquaux@normalesup.org> License: BSD 3 clause OneRClassifier OneRClassifier(resolve_ties='first') OneR (One Rule) Classifier. Parameters resolve_ties : str (default: 'first') Option for how to resolve ties if two or more features have the same error. Options are - 'first' (default): chooses first feature in the list, i.e., feature with the lower column index. - 'chi-squared': performs a chi-squared test for each feature against the target and selects the feature with the lowest p-value. Attributes self.classes_labels_ : array-like, shape = [n_labels] Array containing the unique class labels found in the training set. self.feature_idx_ : int The index of the rules' feature based on the column in the training set. self.p_value_ : float The p value for a given feature. Only available after calling fit when the OneR attribute resolve_ties = 'chi-squared' is set. self.prediction_dict_ : dict Dictionary containing information about the feature's (self.feature_idx_) rules and total error. E.g., {'total error': 37, 'rules (value: class)': {0: 0, 1: 2}} means the total error is 37, and the rules are \"if feature value == 0 classify as 0\" and \"if feature value == 1 classify as 2\". (And classify as class 1 otherwise.) For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/classifier/OneRClassifier/ Methods fit(X, y) Learn rule from training data. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values. Returns self : object get_params(deep=True) Get parameters for this estimator. Parameters deep : bool, default=True If True, will return the parameters for this estimator and contained subobjects that are estimators. Returns params : dict Parameter names mapped to their values. predict(X) Predict class labels for X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns maj : array-like, shape = [n_samples] Predicted class labels. score(X, y, sample_weight=None) Return the mean accuracy on the given test data and labels. In multi-label classification, this is the subset accuracy which is a harsh metric since you require for each sample that each label set be correctly predicted. Parameters X : array-like of shape (n_samples, n_features) Test samples. y : array-like of shape (n_samples,) or (n_samples, n_outputs) True labels for X . sample_weight : array-like of shape (n_samples,), default=None Sample weights. Returns score : float Mean accuracy of self.predict(X) w.r.t. y . set_params( params) Set the parameters of this estimator. The method works on simple estimators as well as on nested objects (such as :class:`~sklearn.pipeline.Pipeline`). The latter have parameters of the form ``<component>__<parameter>`` so that it's possible to update each component of a nested object. Parameters **params : dict Estimator parameters. Returns self : estimator instance Estimator instance. Perceptron Perceptron(eta=0.1, epochs=50, random_seed=None, print_progress=0) Perceptron classifier. Note that this implementation of the Perceptron expects binary class labels in {0, 1}. Parameters eta : float (default: 0.1) Learning rate (between 0.0 and 1.0) epochs : int (default: 50) Number of passes over the training dataset. Prior to each epoch, the dataset is shuffled to prevent cycles. random_seed : int Random state for initializing random weights and shuffling. print_progress : int (default: 0) Prints progress in fitting to stderr. 0: No output 1: Epochs elapsed and cost 2: 1 plus time elapsed 3: 2 plus estimated time until completion Attributes w_ : 2d-array, shape={n_features, 1} Model weights after fitting. b_ : 1d-array, shape={1,} Bias unit after fitting. cost_ : list Number of misclassifications in every epoch. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/classifier/Perceptron/ Methods fit(X, y, init_params=True) Learn model from training data. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values. init_params : bool (default: True) Re-initializes model parameters prior to fitting. Set False to continue training with weights from a previous model fitting. Returns self : object get_params(deep=True) Get parameters for this estimator. Parameters deep : boolean, optional If True, will return the parameters for this estimator and contained subobjects that are estimators. Returns params : mapping of string to any Parameter names mapped to their values.' adapted from https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/base.py Author: Gael Varoquaux gael.varoquaux@normalesup.org License: BSD 3 clause predict(X) Predict targets from X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns target_values : array-like, shape = [n_samples] Predicted target values. score(X, y) Compute the prediction accuracy Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values (true class labels). Returns acc : float The prediction accuracy as a float between 0.0 and 1.0 (perfect score). set_params( params) Set the parameters of this estimator. The method works on simple estimators as well as on nested objects (such as pipelines). The latter have parameters of the form <component>__<parameter> so that it's possible to update each component of a nested object. Returns self adapted from https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/base.py Author: Gael Varoquaux <gael.varoquaux@normalesup.org> License: BSD 3 clause SoftmaxRegression SoftmaxRegression(eta=0.01, epochs=50, l2=0.0, minibatches=1, n_classes=None, random_seed=None, print_progress=0) Softmax regression classifier. Parameters eta : float (default: 0.01) Learning rate (between 0.0 and 1.0) epochs : int (default: 50) Passes over the training dataset. Prior to each epoch, the dataset is shuffled if minibatches > 1 to prevent cycles in stochastic gradient descent. l2 : float Regularization parameter for L2 regularization. No regularization if l2=0.0. minibatches : int (default: 1) The number of minibatches for gradient-based optimization. If 1: Gradient Descent learning If len(y): Stochastic Gradient Descent (SGD) online learning If 1 < minibatches < len(y): SGD Minibatch learning n_classes : int (default: None) A positive integer to declare the number of class labels if not all class labels are present in a partial training set. Gets the number of class labels automatically if None. random_seed : int (default: None) Set random state for shuffling and initializing the weights. print_progress : int (default: 0) Prints progress in fitting to stderr. 0: No output 1: Epochs elapsed and cost 2: 1 plus time elapsed 3: 2 plus estimated time until completion Attributes w_ : 2d-array, shape={n_features, 1} Model weights after fitting. b_ : 1d-array, shape={1,} Bias unit after fitting. cost_ : list List of floats, the average cross_entropy for each epoch. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/classifier/SoftmaxRegression/ Methods fit(X, y, init_params=True) Learn model from training data. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values. init_params : bool (default: True) Re-initializes model parameters prior to fitting. Set False to continue training with weights from a previous model fitting. Returns self : object get_params(deep=True) Get parameters for this estimator. Parameters deep : boolean, optional If True, will return the parameters for this estimator and contained subobjects that are estimators. Returns params : mapping of string to any Parameter names mapped to their values.' adapted from https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/base.py Author: Gael Varoquaux gael.varoquaux@normalesup.org License: BSD 3 clause predict(X) Predict targets from X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns target_values : array-like, shape = [n_samples] Predicted target values. predict_proba(X) Predict class probabilities of X from the net input. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns Class probabilties : array-like, shape= [n_samples, n_classes] score(X, y) Compute the prediction accuracy Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values (true class labels). Returns acc : float The prediction accuracy as a float between 0.0 and 1.0 (perfect score). set_params( params) Set the parameters of this estimator. The method works on simple estimators as well as on nested objects (such as pipelines). The latter have parameters of the form <component>__<parameter> so that it's possible to update each component of a nested object. Returns self adapted from https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/base.py Author: Gael Varoquaux <gael.varoquaux@normalesup.org> License: BSD 3 clause StackingCVClassifier StackingCVClassifier(classifiers, meta_classifier, use_probas=False, drop_proba_col=None, cv=2, shuffle=True, random_state=None, stratify=True, verbose=0, use_features_in_secondary=False, store_train_meta_features=False, use_clones=True, n_jobs=None, pre_dispatch='2 n_jobs')* A 'Stacking Cross-Validation' classifier for scikit-learn estimators. New in mlxtend v0.4.3 Parameters classifiers : array-like, shape = [n_classifiers] A list of classifiers. Invoking the fit method on the StackingCVClassifer will fit clones of these original classifiers that will be stored in the class attribute self.clfs_ if use_clones=True . meta_classifier : object The meta-classifier to be fitted on the ensemble of classifiers use_probas : bool (default: False) If True, trains meta-classifier based on predicted probabilities instead of class labels. drop_proba_col : string (default: None) Drops extra \"probability\" column in the feature set, because it is redundant: p(y_c) = 1 - p(y_1) + p(y_2) + ... + p(y_{c-1}). This can be useful for meta-classifiers that are sensitive to perfectly collinear features. If 'last', drops last probability column. If 'first', drops first probability column. Only relevant if use_probas=True . cv : int, cross-validation generator or an iterable, optional (default: 2) Determines the cross-validation splitting strategy. Possible inputs for cv are: - None, to use the default 2-fold cross validation, - integer, to specify the number of folds in a (Stratified)KFold , - An object to be used as a cross-validation generator. - An iterable yielding train, test splits. For integer/None inputs, it will use either a KFold or StratifiedKFold cross validation depending the value of stratify argument. shuffle : bool (default: True) If True, and the cv argument is integer, the training data will be shuffled at fitting stage prior to cross-validation. If the cv argument is a specific cross validation technique, this argument is omitted. random_state : int, RandomState instance or None, optional (default: None) Constrols the randomness of the cv splitter. Used when cv is integer and shuffle=True . New in v0.16.0. stratify : bool (default: True) If True, and the cv argument is integer it will follow a stratified K-Fold cross validation technique. If the cv argument is a specific cross validation technique, this argument is omitted. verbose : int, optional (default=0) Controls the verbosity of the building process. - verbose=0 (default): Prints nothing - verbose=1 : Prints the number & name of the regressor being fitted and which fold is currently being used for fitting - verbose=2 : Prints info about the parameters of the regressor being fitted - verbose>2 : Changes verbose param of the underlying regressor to self.verbose - 2 use_features_in_secondary : bool (default: False) If True, the meta-classifier will be trained both on the predictions of the original classifiers and the original dataset. If False, the meta-classifier will be trained only on the predictions of the original classifiers. store_train_meta_features : bool (default: False) If True, the meta-features computed from the training data used for fitting the meta-classifier stored in the self.train_meta_features_ array, which can be accessed after calling fit . use_clones : bool (default: True) Clones the classifiers for stacking classification if True (default) or else uses the original ones, which will be refitted on the dataset upon calling the fit method. Hence, if use_clones=True, the original input classifiers will remain unmodified upon using the StackingCVClassifier's fit method. Setting use_clones=False is recommended if you are working with estimators that are supporting the scikit-learn fit/predict API interface but are not compatible to scikit-learn's clone function. n_jobs : int or None, optional (default=None) The number of CPUs to use to do the computation. None means 1 unless in a joblib.parallel_backend context. -1 means using all processors. for more details. New in v0.16.0. pre_dispatch : int, or string, optional Controls the number of jobs that get dispatched during parallel execution. Reducing this number can be useful to avoid an explosion of memory consumption when more jobs get dispatched than CPUs can process. This parameter can be: - None, in which case all the jobs are immediately created and spawned. Use this for lightweight and fast-running jobs, to avoid delays due to on-demand spawning of the jobs - An int, giving the exact number of total jobs that are spawned - A string, giving an expression as a function of n_jobs, as in '2*n_jobs' New in v0.16.0. Attributes clfs_ : list, shape=[n_classifiers] Fitted classifiers (clones of the original classifiers) meta_clf_ : estimator Fitted meta-classifier (clone of the original meta-estimator) train_meta_features : numpy array, shape = [n_samples, n_classifiers] meta-features for training data, where n_samples is the number of samples in training data and n_classifiers is the number of classfiers. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/classifier/StackingCVClassifier/ Methods decision_function(X) Predict class confidence scores for X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns scores : shape=(n_samples,) if n_classes == 2 else (n_samples, n_classes). Confidence scores per (sample, class) combination. In the binary case, confidence score for self.classes_[1] where >0 means this class would be predicted. fit(X, y, groups=None, sample_weight=None) Fit ensemble classifers and the meta-classifier. Parameters X : numpy array, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : numpy array, shape = [n_samples] Target values. groups : numpy array/None, shape = [n_samples] The group that each sample belongs to. This is used by specific folding strategies such as GroupKFold() sample_weight : array-like, shape = [n_samples], optional Sample weights passed as sample_weights to each regressor in the regressors list as well as the meta_regressor. Raises error if some regressor does not support sample_weight in the fit() method. Returns self : object fit_transform(X, y=None, fit_params) Fit to data, then transform it. Fits transformer to `X` and `y` with optional parameters `fit_params` and returns a transformed version of `X`. Parameters X : array-like of shape (n_samples, n_features) Input samples. y : array-like of shape (n_samples,) or (n_samples, n_outputs), default=None Target values (None for unsupervised transformations). **fit_params : dict Additional fit parameters. Returns X_new : ndarray array of shape (n_samples, n_features_new) Transformed array. get_params(deep=True) Return estimator parameter names for GridSearch support. predict(X) Predict target values for X. Parameters X : numpy array, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns labels : array-like, shape = [n_samples] Predicted class labels. predict_meta_features(X) Get meta-features of test-data. Parameters X : numpy array, shape = [n_samples, n_features] Test vectors, where n_samples is the number of samples and n_features is the number of features. Returns meta-features : numpy array, shape = [n_samples, n_classifiers] Returns the meta-features for test data. predict_proba(X) Predict class probabilities for X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns proba : array-like, shape = [n_samples, n_classes] or a list of n_outputs of such arrays if n_outputs > 1. Probability for each class per sample. score(X, y, sample_weight=None) Return the mean accuracy on the given test data and labels. In multi-label classification, this is the subset accuracy which is a harsh metric since you require for each sample that each label set be correctly predicted. Parameters X : array-like of shape (n_samples, n_features) Test samples. y : array-like of shape (n_samples,) or (n_samples, n_outputs) True labels for X . sample_weight : array-like of shape (n_samples,), default=None Sample weights. Returns score : float Mean accuracy of self.predict(X) w.r.t. y . set_output( , transform=None)* Set output container. See :ref:`sphx_glr_auto_examples_miscellaneous_plot_set_output.py` for an example on how to use the API. Parameters transform : {\"default\", \"pandas\"}, default=None Configure output of transform and fit_transform . \"default\" : Default output format of a transformer \"pandas\" : DataFrame output None : Transform configuration is unchanged Returns self : estimator instance Estimator instance. set_params( params) Set the parameters of this estimator. Valid parameter keys can be listed with ``get_params()``. Returns self Properties named_classifiers None StackingClassifier StackingClassifier(classifiers, meta_classifier, use_probas=False, drop_proba_col=None, average_probas=False, verbose=0, use_features_in_secondary=False, store_train_meta_features=False, use_clones=True, fit_base_estimators=True) A Stacking classifier for scikit-learn estimators for classification. Parameters classifiers : array-like, shape = [n_classifiers] A list of classifiers. Invoking the fit method on the StackingClassifer will fit clones of these original classifiers that will be stored in the class attribute self.clfs_ if use_clones=True (default) and fit_base_estimators=True (default). meta_classifier : object The meta-classifier to be fitted on the ensemble of classifiers use_probas : bool (default: False) If True, trains meta-classifier based on predicted probabilities instead of class labels. drop_proba_col : string (default: None) Drops extra \"probability\" column in the feature set, because it is redundant: p(y_c) = 1 - p(y_1) + p(y_2) + ... + p(y_{c-1}). This can be useful for meta-classifiers that are sensitive to perfectly collinear features. If 'last', drops last probability column. If 'first', drops first probability column. Only relevant if use_probas=True . average_probas : bool (default: False) Averages the probabilities as meta features if True . Only relevant if use_probas=True . verbose : int, optional (default=0) Controls the verbosity of the building process. - verbose=0 (default): Prints nothing - verbose=1 : Prints the number & name of the regressor being fitted - verbose=2 : Prints info about the parameters of the regressor being fitted - verbose>2 : Changes verbose param of the underlying regressor to self.verbose - 2 use_features_in_secondary : bool (default: False) If True, the meta-classifier will be trained both on the predictions of the original classifiers and the original dataset. If False, the meta-classifier will be trained only on the predictions of the original classifiers. store_train_meta_features : bool (default: False) If True, the meta-features computed from the training data used for fitting the meta-classifier stored in the self.train_meta_features_ array, which can be accessed after calling fit . use_clones : bool (default: True) Clones the classifiers for stacking classification if True (default) or else uses the original ones, which will be refitted on the dataset upon calling the fit method. Hence, if use_clones=True, the original input classifiers will remain unmodified upon using the StackingClassifier's fit method. Setting use_clones=False is recommended if you are working with estimators that are supporting the scikit-learn fit/predict API interface but are not compatible to scikit-learn's clone function. fit_base_estimators: bool (default: True) Refits classifiers in classifiers if True; uses references to the classifiers , otherwise (assumes that the classifiers were already fit). Note: fit_base_estimators=False will enforce use_clones to be False, and is incompatible to most scikit-learn wrappers! For instance, if any form of cross-validation is performed this would require the re-fitting classifiers to training folds, which would raise a NotFitterError if fit_base_estimators=False. (New in mlxtend v0.6.) Attributes clfs_ : list, shape=[n_classifiers] Fitted classifiers (clones of the original classifiers) meta_clf_ : estimator Fitted meta-classifier (clone of the original meta-estimator) train_meta_features : numpy array, shape = [n_samples, n_classifiers] meta-features for training data, where n_samples is the number of samples in training data and n_classifiers is the number of classfiers. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/classifier/StackingClassifier/ Methods decision_function(X) Predict class confidence scores for X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns scores : shape=(n_samples,) if n_classes == 2 else (n_samples, n_classes). Confidence scores per (sample, class) combination. In the binary case, confidence score for self.classes_[1] where >0 means this class would be predicted. fit(X, y, sample_weight=None) Fit ensemble classifers and the meta-classifier. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] or [n_samples, n_outputs] Target values. sample_weight : array-like, shape = [n_samples], optional Sample weights passed as sample_weights to each regressor in the regressors list as well as the meta_regressor. Raises error if some regressor does not support sample_weight in the fit() method. Returns self : object fit_transform(X, y=None, fit_params) Fit to data, then transform it. Fits transformer to `X` and `y` with optional parameters `fit_params` and returns a transformed version of `X`. Parameters X : array-like of shape (n_samples, n_features) Input samples. y : array-like of shape (n_samples,) or (n_samples, n_outputs), default=None Target values (None for unsupervised transformations). **fit_params : dict Additional fit parameters. Returns X_new : ndarray array of shape (n_samples, n_features_new) Transformed array. get_params(deep=True) Return estimator parameter names for GridSearch support. predict(X) Predict target values for X. Parameters X : numpy array, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns labels : array-like, shape = [n_samples] Predicted class labels. predict_meta_features(X) Get meta-features of test-data. Parameters X : numpy array, shape = [n_samples, n_features] Test vectors, where n_samples is the number of samples and n_features is the number of features. Returns meta-features : numpy array, shape = [n_samples, n_classifiers] Returns the meta-features for test data. predict_proba(X) Predict class probabilities for X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns proba : array-like, shape = [n_samples, n_classes] or a list of n_outputs of such arrays if n_outputs > 1. Probability for each class per sample. score(X, y, sample_weight=None) Return the mean accuracy on the given test data and labels. In multi-label classification, this is the subset accuracy which is a harsh metric since you require for each sample that each label set be correctly predicted. Parameters X : array-like of shape (n_samples, n_features) Test samples. y : array-like of shape (n_samples,) or (n_samples, n_outputs) True labels for X . sample_weight : array-like of shape (n_samples,), default=None Sample weights. Returns score : float Mean accuracy of self.predict(X) w.r.t. y . set_output( , transform=None)* Set output container. See :ref:`sphx_glr_auto_examples_miscellaneous_plot_set_output.py` for an example on how to use the API. Parameters transform : {\"default\", \"pandas\"}, default=None Configure output of transform and fit_transform . \"default\" : Default output format of a transformer \"pandas\" : DataFrame output None : Transform configuration is unchanged Returns self : estimator instance Estimator instance. set_params( params) Set the parameters of this estimator. Valid parameter keys can be listed with ``get_params()``. Returns self Properties named_classifiers None","title":"Mlxtend.classifier"},{"location":"api_subpackages/mlxtend.classifier/#adaline","text":"Adaline(eta=0.01, epochs=50, minibatches=None, random_seed=None, print_progress=0) ADAptive LInear NEuron classifier. Note that this implementation of Adaline expects binary class labels in {0, 1}. Parameters eta : float (default: 0.01) solver rate (between 0.0 and 1.0) epochs : int (default: 50) Passes over the training dataset. Prior to each epoch, the dataset is shuffled if minibatches > 1 to prevent cycles in stochastic gradient descent. minibatches : int (default: None) The number of minibatches for gradient-based optimization. If None: Normal Equations (closed-form solution) If 1: Gradient Descent learning If len(y): Stochastic Gradient Descent (SGD) online learning If 1 < minibatches < len(y): SGD Minibatch learning random_seed : int (default: None) Set random state for shuffling and initializing the weights. print_progress : int (default: 0) Prints progress in fitting to stderr if not solver='normal equation' 0: No output 1: Epochs elapsed and cost 2: 1 plus time elapsed 3: 2 plus estimated time until completion Attributes w_ : 2d-array, shape={n_features, 1} Model weights after fitting. b_ : 1d-array, shape={1,} Bias unit after fitting. cost_ : list Sum of squared errors after each epoch. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/classifier/Adaline/","title":"Adaline"},{"location":"api_subpackages/mlxtend.classifier/#methods","text":"fit(X, y, init_params=True) Learn model from training data. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values. init_params : bool (default: True) Re-initializes model parameters prior to fitting. Set False to continue training with weights from a previous model fitting. Returns self : object get_params(deep=True) Get parameters for this estimator. Parameters deep : boolean, optional If True, will return the parameters for this estimator and contained subobjects that are estimators. Returns params : mapping of string to any Parameter names mapped to their values.' adapted from https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/base.py Author: Gael Varoquaux gael.varoquaux@normalesup.org License: BSD 3 clause predict(X) Predict targets from X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns target_values : array-like, shape = [n_samples] Predicted target values. score(X, y) Compute the prediction accuracy Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values (true class labels). Returns acc : float The prediction accuracy as a float between 0.0 and 1.0 (perfect score). set_params( params) Set the parameters of this estimator. The method works on simple estimators as well as on nested objects (such as pipelines). The latter have parameters of the form <component>__<parameter> so that it's possible to update each component of a nested object. Returns self adapted from https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/base.py Author: Gael Varoquaux <gael.varoquaux@normalesup.org> License: BSD 3 clause","title":"Methods"},{"location":"api_subpackages/mlxtend.classifier/#ensemblevoteclassifier","text":"EnsembleVoteClassifier(clfs, voting='hard', weights=None, verbose=0, use_clones=True, fit_base_estimators=True) Soft Voting/Majority Rule classifier for scikit-learn estimators. Parameters clfs : array-like, shape = [n_classifiers] A list of classifiers. Invoking the fit method on the VotingClassifier will fit clones of those original classifiers be stored in the class attribute if use_clones=True (default) and fit_base_estimators=True (default). voting : str, {'hard', 'soft'} (default='hard') If 'hard', uses predicted class labels for majority rule voting. Else if 'soft', predicts the class label based on the argmax of the sums of the predicted probalities, which is recommended for an ensemble of well-calibrated classifiers. weights : array-like, shape = [n_classifiers], optional (default= None ) Sequence of weights ( float or int ) to weight the occurances of predicted class labels ( hard voting) or class probabilities before averaging ( soft voting). Uses uniform weights if None . verbose : int, optional (default=0) Controls the verbosity of the building process. - verbose=0 (default): Prints nothing - verbose=1 : Prints the number & name of the clf being fitted - verbose=2 : Prints info about the parameters of the clf being fitted - verbose>2 : Changes verbose param of the underlying clf to self.verbose - 2 use_clones : bool (default: True) Clones the classifiers for stacking classification if True (default) or else uses the original ones, which will be refitted on the dataset upon calling the fit method. Hence, if use_clones=True, the original input classifiers will remain unmodified upon using the StackingClassifier's fit method. Setting use_clones=False is recommended if you are working with estimators that are supporting the scikit-learn fit/predict API interface but are not compatible to scikit-learn's clone function. fit_base_estimators : bool (default: True) Refits classifiers in clfs if True; uses references to the clfs , otherwise (assumes that the classifiers were already fit). Note: fit_base_estimators=False will enforce use_clones to be False, and is incompatible to most scikit-learn wrappers! For instance, if any form of cross-validation is performed this would require the re-fitting classifiers to training folds, which would raise a NotFitterError if fit_base_estimators=False. (New in mlxtend v0.6.) Attributes classes_ : array-like, shape = [n_predictions] clf : array-like, shape = [n_predictions] The input classifiers; may be overwritten if use_clones=False clf_ : array-like, shape = [n_predictions] Fitted input classifiers; clones if use_clones=True Examples ``` >>> import numpy as np >>> from sklearn.linear_model import LogisticRegression >>> from sklearn.naive_bayes import GaussianNB >>> from sklearn.ensemble import RandomForestClassifier >>> from mlxtend.sklearn import EnsembleVoteClassifier >>> clf1 = LogisticRegression(random_seed=1) >>> clf2 = RandomForestClassifier(random_seed=1) >>> clf3 = GaussianNB() >>> X = np.array([[-1, -1], [-2, -1], [-3, -2], [1, 1], [2, 1], [3, 2]]) >>> y = np.array([1, 1, 1, 2, 2, 2]) >>> eclf1 = EnsembleVoteClassifier(clfs=[clf1, clf2, clf3], ... voting='hard', verbose=1) >>> eclf1 = eclf1.fit(X, y) >>> print(eclf1.predict(X)) [1 1 1 2 2 2] >>> eclf2 = EnsembleVoteClassifier(clfs=[clf1, clf2, clf3], voting='soft') >>> eclf2 = eclf2.fit(X, y) >>> print(eclf2.predict(X)) [1 1 1 2 2 2] >>> eclf3 = EnsembleVoteClassifier(clfs=[clf1, clf2, clf3], ... voting='soft', weights=[2,1,1]) >>> eclf3 = eclf3.fit(X, y) >>> print(eclf3.predict(X)) [1 1 1 2 2 2] >>> For more usage examples, please see https://rasbt.github.io/mlxtend/user_guide/classifier/EnsembleVoteClassifier/ ```","title":"EnsembleVoteClassifier"},{"location":"api_subpackages/mlxtend.classifier/#methods_1","text":"fit(X, y, sample_weight=None) Learn weight coefficients from training data for each classifier. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values. sample_weight : array-like, shape = [n_samples], optional Sample weights passed as sample_weights to each regressor in the regressors list as well as the meta_regressor. Raises error if some regressor does not support sample_weight in the fit() method. Returns self : object fit_transform(X, y=None, fit_params) Fit to data, then transform it. Fits transformer to `X` and `y` with optional parameters `fit_params` and returns a transformed version of `X`. Parameters X : array-like of shape (n_samples, n_features) Input samples. y : array-like of shape (n_samples,) or (n_samples, n_outputs), default=None Target values (None for unsupervised transformations). **fit_params : dict Additional fit parameters. Returns X_new : ndarray array of shape (n_samples, n_features_new) Transformed array. get_params(deep=True) Return estimator parameter names for GridSearch support. predict(X) Predict class labels for X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns maj : array-like, shape = [n_samples] Predicted class labels. predict_proba(X) Predict class probabilities for X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns avg : array-like, shape = [n_samples, n_classes] Weighted average probability for each class per sample. score(X, y, sample_weight=None) Return the mean accuracy on the given test data and labels. In multi-label classification, this is the subset accuracy which is a harsh metric since you require for each sample that each label set be correctly predicted. Parameters X : array-like of shape (n_samples, n_features) Test samples. y : array-like of shape (n_samples,) or (n_samples, n_outputs) True labels for X . sample_weight : array-like of shape (n_samples,), default=None Sample weights. Returns score : float Mean accuracy of self.predict(X) w.r.t. y . set_output( , transform=None)* Set output container. See :ref:`sphx_glr_auto_examples_miscellaneous_plot_set_output.py` for an example on how to use the API. Parameters transform : {\"default\", \"pandas\"}, default=None Configure output of transform and fit_transform . \"default\" : Default output format of a transformer \"pandas\" : DataFrame output None : Transform configuration is unchanged Returns self : estimator instance Estimator instance. set_params( params) Set the parameters of this estimator. The method works on simple estimators as well as on nested objects (such as :class:`~sklearn.pipeline.Pipeline`). The latter have parameters of the form ``<component>__<parameter>`` so that it's possible to update each component of a nested object. Parameters **params : dict Estimator parameters. Returns self : estimator instance Estimator instance. transform(X) Return class labels or probabilities for X for each estimator. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns If voting='soft'`` : array-like = [n_classifiers, n_samples, n_classes] Class probabilties calculated by each classifier. If voting='hard'`` : array-like = [n_classifiers, n_samples] Class labels predicted by each classifier.","title":"Methods"},{"location":"api_subpackages/mlxtend.classifier/#logisticregression","text":"LogisticRegression(eta=0.01, epochs=50, l2_lambda=0.0, minibatches=1, random_seed=None, print_progress=0) Logistic regression classifier. Note that this implementation of Logistic Regression expects binary class labels in {0, 1}. Parameters eta : float (default: 0.01) Learning rate (between 0.0 and 1.0) epochs : int (default: 50) Passes over the training dataset. Prior to each epoch, the dataset is shuffled if minibatches > 1 to prevent cycles in stochastic gradient descent. l2_lambda : float Regularization parameter for L2 regularization. No regularization if l2_lambda=0.0. minibatches : int (default: 1) The number of minibatches for gradient-based optimization. If 1: Gradient Descent learning If len(y): Stochastic Gradient Descent (SGD) online learning If 1 < minibatches < len(y): SGD Minibatch learning random_seed : int (default: None) Set random state for shuffling and initializing the weights. print_progress : int (default: 0) Prints progress in fitting to stderr. 0: No output 1: Epochs elapsed and cost 2: 1 plus time elapsed 3: 2 plus estimated time until completion Attributes w_ : 2d-array, shape={n_features, 1} Model weights after fitting. b_ : 1d-array, shape={1,} Bias unit after fitting. cost_ : list List of floats with cross_entropy cost (sgd or gd) for every epoch. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/classifier/LogisticRegression/","title":"LogisticRegression"},{"location":"api_subpackages/mlxtend.classifier/#methods_2","text":"fit(X, y, init_params=True) Learn model from training data. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values. init_params : bool (default: True) Re-initializes model parameters prior to fitting. Set False to continue training with weights from a previous model fitting. Returns self : object get_params(deep=True) Get parameters for this estimator. Parameters deep : boolean, optional If True, will return the parameters for this estimator and contained subobjects that are estimators. Returns params : mapping of string to any Parameter names mapped to their values.' adapted from https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/base.py Author: Gael Varoquaux gael.varoquaux@normalesup.org License: BSD 3 clause predict(X) Predict targets from X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns target_values : array-like, shape = [n_samples] Predicted target values. predict_proba(X) Predict class probabilities of X from the net input. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns Class 1 probability : float score(X, y) Compute the prediction accuracy Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values (true class labels). Returns acc : float The prediction accuracy as a float between 0.0 and 1.0 (perfect score). set_params( params) Set the parameters of this estimator. The method works on simple estimators as well as on nested objects (such as pipelines). The latter have parameters of the form <component>__<parameter> so that it's possible to update each component of a nested object. Returns self adapted from https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/base.py Author: Gael Varoquaux <gael.varoquaux@normalesup.org> License: BSD 3 clause","title":"Methods"},{"location":"api_subpackages/mlxtend.classifier/#multilayerperceptron","text":"MultiLayerPerceptron(eta=0.5, epochs=50, hidden_layers=[50], n_classes=None, momentum=0.0, l1=0.0, l2=0.0, dropout=1.0, decrease_const=0.0, minibatches=1, random_seed=None, print_progress=0) Multi-layer perceptron classifier with logistic sigmoid activations Parameters eta : float (default: 0.5) Learning rate (between 0.0 and 1.0) epochs : int (default: 50) Passes over the training dataset. Prior to each epoch, the dataset is shuffled if minibatches > 1 to prevent cycles in stochastic gradient descent. hidden_layers : list (default: [50]) Number of units per hidden layer. By default 50 units in the first hidden layer. At the moment only 1 hidden layer is supported n_classes : int (default: None) A positive integer to declare the number of class labels if not all class labels are present in a partial training set. Gets the number of class labels automatically if None. l1 : float (default: 0.0) L1 regularization strength l2 : float (default: 0.0) L2 regularization strength momentum : float (default: 0.0) Momentum constant. Factor multiplied with the gradient of the previous epoch t-1 to improve learning speed w(t) := w(t) - (grad(t) + momentum * grad(t-1)) decrease_const : float (default: 0.0) Decrease constant. Shrinks the learning rate after each epoch via eta / (1 + epoch*decrease_const) minibatches : int (default: 1) Divide the training data into k minibatches for accelerated stochastic gradient descent learning. Gradient Descent Learning if minibatches = 1 Stochastic Gradient Descent learning if minibatches = len(y) Minibatch learning if minibatches > 1 random_seed : int (default: None) Set random state for shuffling and initializing the weights. print_progress : int (default: 0) Prints progress in fitting to stderr. 0: No output 1: Epochs elapsed and cost 2: 1 plus time elapsed 3: 2 plus estimated time until completion Attributes w_ : 2d-array, shape=[n_features, n_classes] Weights after fitting. b_ : 1D-array, shape=[n_classes] Bias units after fitting. cost_ : list List of floats; the mean categorical cross entropy cost after each epoch. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/classifier/MultiLayerPerceptron/","title":"MultiLayerPerceptron"},{"location":"api_subpackages/mlxtend.classifier/#methods_3","text":"fit(X, y, init_params=True) Learn model from training data. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values. init_params : bool (default: True) Re-initializes model parameters prior to fitting. Set False to continue training with weights from a previous model fitting. Returns self : object get_params(deep=True) Get parameters for this estimator. Parameters deep : boolean, optional If True, will return the parameters for this estimator and contained subobjects that are estimators. Returns params : mapping of string to any Parameter names mapped to their values.' adapted from https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/base.py Author: Gael Varoquaux gael.varoquaux@normalesup.org License: BSD 3 clause predict(X) Predict targets from X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns target_values : array-like, shape = [n_samples] Predicted target values. predict_proba(X) Predict class probabilities of X from the net input. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns Class probabilties : array-like, shape= [n_samples, n_classes] score(X, y) Compute the prediction accuracy Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values (true class labels). Returns acc : float The prediction accuracy as a float between 0.0 and 1.0 (perfect score). set_params( params) Set the parameters of this estimator. The method works on simple estimators as well as on nested objects (such as pipelines). The latter have parameters of the form <component>__<parameter> so that it's possible to update each component of a nested object. Returns self adapted from https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/base.py Author: Gael Varoquaux <gael.varoquaux@normalesup.org> License: BSD 3 clause","title":"Methods"},{"location":"api_subpackages/mlxtend.classifier/#onerclassifier","text":"OneRClassifier(resolve_ties='first') OneR (One Rule) Classifier. Parameters resolve_ties : str (default: 'first') Option for how to resolve ties if two or more features have the same error. Options are - 'first' (default): chooses first feature in the list, i.e., feature with the lower column index. - 'chi-squared': performs a chi-squared test for each feature against the target and selects the feature with the lowest p-value. Attributes self.classes_labels_ : array-like, shape = [n_labels] Array containing the unique class labels found in the training set. self.feature_idx_ : int The index of the rules' feature based on the column in the training set. self.p_value_ : float The p value for a given feature. Only available after calling fit when the OneR attribute resolve_ties = 'chi-squared' is set. self.prediction_dict_ : dict Dictionary containing information about the feature's (self.feature_idx_) rules and total error. E.g., {'total error': 37, 'rules (value: class)': {0: 0, 1: 2}} means the total error is 37, and the rules are \"if feature value == 0 classify as 0\" and \"if feature value == 1 classify as 2\". (And classify as class 1 otherwise.) For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/classifier/OneRClassifier/","title":"OneRClassifier"},{"location":"api_subpackages/mlxtend.classifier/#methods_4","text":"fit(X, y) Learn rule from training data. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values. Returns self : object get_params(deep=True) Get parameters for this estimator. Parameters deep : bool, default=True If True, will return the parameters for this estimator and contained subobjects that are estimators. Returns params : dict Parameter names mapped to their values. predict(X) Predict class labels for X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns maj : array-like, shape = [n_samples] Predicted class labels. score(X, y, sample_weight=None) Return the mean accuracy on the given test data and labels. In multi-label classification, this is the subset accuracy which is a harsh metric since you require for each sample that each label set be correctly predicted. Parameters X : array-like of shape (n_samples, n_features) Test samples. y : array-like of shape (n_samples,) or (n_samples, n_outputs) True labels for X . sample_weight : array-like of shape (n_samples,), default=None Sample weights. Returns score : float Mean accuracy of self.predict(X) w.r.t. y . set_params( params) Set the parameters of this estimator. The method works on simple estimators as well as on nested objects (such as :class:`~sklearn.pipeline.Pipeline`). The latter have parameters of the form ``<component>__<parameter>`` so that it's possible to update each component of a nested object. Parameters **params : dict Estimator parameters. Returns self : estimator instance Estimator instance.","title":"Methods"},{"location":"api_subpackages/mlxtend.classifier/#perceptron","text":"Perceptron(eta=0.1, epochs=50, random_seed=None, print_progress=0) Perceptron classifier. Note that this implementation of the Perceptron expects binary class labels in {0, 1}. Parameters eta : float (default: 0.1) Learning rate (between 0.0 and 1.0) epochs : int (default: 50) Number of passes over the training dataset. Prior to each epoch, the dataset is shuffled to prevent cycles. random_seed : int Random state for initializing random weights and shuffling. print_progress : int (default: 0) Prints progress in fitting to stderr. 0: No output 1: Epochs elapsed and cost 2: 1 plus time elapsed 3: 2 plus estimated time until completion Attributes w_ : 2d-array, shape={n_features, 1} Model weights after fitting. b_ : 1d-array, shape={1,} Bias unit after fitting. cost_ : list Number of misclassifications in every epoch. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/classifier/Perceptron/","title":"Perceptron"},{"location":"api_subpackages/mlxtend.classifier/#methods_5","text":"fit(X, y, init_params=True) Learn model from training data. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values. init_params : bool (default: True) Re-initializes model parameters prior to fitting. Set False to continue training with weights from a previous model fitting. Returns self : object get_params(deep=True) Get parameters for this estimator. Parameters deep : boolean, optional If True, will return the parameters for this estimator and contained subobjects that are estimators. Returns params : mapping of string to any Parameter names mapped to their values.' adapted from https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/base.py Author: Gael Varoquaux gael.varoquaux@normalesup.org License: BSD 3 clause predict(X) Predict targets from X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns target_values : array-like, shape = [n_samples] Predicted target values. score(X, y) Compute the prediction accuracy Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values (true class labels). Returns acc : float The prediction accuracy as a float between 0.0 and 1.0 (perfect score). set_params( params) Set the parameters of this estimator. The method works on simple estimators as well as on nested objects (such as pipelines). The latter have parameters of the form <component>__<parameter> so that it's possible to update each component of a nested object. Returns self adapted from https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/base.py Author: Gael Varoquaux <gael.varoquaux@normalesup.org> License: BSD 3 clause","title":"Methods"},{"location":"api_subpackages/mlxtend.classifier/#softmaxregression","text":"SoftmaxRegression(eta=0.01, epochs=50, l2=0.0, minibatches=1, n_classes=None, random_seed=None, print_progress=0) Softmax regression classifier. Parameters eta : float (default: 0.01) Learning rate (between 0.0 and 1.0) epochs : int (default: 50) Passes over the training dataset. Prior to each epoch, the dataset is shuffled if minibatches > 1 to prevent cycles in stochastic gradient descent. l2 : float Regularization parameter for L2 regularization. No regularization if l2=0.0. minibatches : int (default: 1) The number of minibatches for gradient-based optimization. If 1: Gradient Descent learning If len(y): Stochastic Gradient Descent (SGD) online learning If 1 < minibatches < len(y): SGD Minibatch learning n_classes : int (default: None) A positive integer to declare the number of class labels if not all class labels are present in a partial training set. Gets the number of class labels automatically if None. random_seed : int (default: None) Set random state for shuffling and initializing the weights. print_progress : int (default: 0) Prints progress in fitting to stderr. 0: No output 1: Epochs elapsed and cost 2: 1 plus time elapsed 3: 2 plus estimated time until completion Attributes w_ : 2d-array, shape={n_features, 1} Model weights after fitting. b_ : 1d-array, shape={1,} Bias unit after fitting. cost_ : list List of floats, the average cross_entropy for each epoch. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/classifier/SoftmaxRegression/","title":"SoftmaxRegression"},{"location":"api_subpackages/mlxtend.classifier/#methods_6","text":"fit(X, y, init_params=True) Learn model from training data. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values. init_params : bool (default: True) Re-initializes model parameters prior to fitting. Set False to continue training with weights from a previous model fitting. Returns self : object get_params(deep=True) Get parameters for this estimator. Parameters deep : boolean, optional If True, will return the parameters for this estimator and contained subobjects that are estimators. Returns params : mapping of string to any Parameter names mapped to their values.' adapted from https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/base.py Author: Gael Varoquaux gael.varoquaux@normalesup.org License: BSD 3 clause predict(X) Predict targets from X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns target_values : array-like, shape = [n_samples] Predicted target values. predict_proba(X) Predict class probabilities of X from the net input. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns Class probabilties : array-like, shape= [n_samples, n_classes] score(X, y) Compute the prediction accuracy Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values (true class labels). Returns acc : float The prediction accuracy as a float between 0.0 and 1.0 (perfect score). set_params( params) Set the parameters of this estimator. The method works on simple estimators as well as on nested objects (such as pipelines). The latter have parameters of the form <component>__<parameter> so that it's possible to update each component of a nested object. Returns self adapted from https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/base.py Author: Gael Varoquaux <gael.varoquaux@normalesup.org> License: BSD 3 clause","title":"Methods"},{"location":"api_subpackages/mlxtend.classifier/#stackingcvclassifier","text":"StackingCVClassifier(classifiers, meta_classifier, use_probas=False, drop_proba_col=None, cv=2, shuffle=True, random_state=None, stratify=True, verbose=0, use_features_in_secondary=False, store_train_meta_features=False, use_clones=True, n_jobs=None, pre_dispatch='2 n_jobs')* A 'Stacking Cross-Validation' classifier for scikit-learn estimators. New in mlxtend v0.4.3 Parameters classifiers : array-like, shape = [n_classifiers] A list of classifiers. Invoking the fit method on the StackingCVClassifer will fit clones of these original classifiers that will be stored in the class attribute self.clfs_ if use_clones=True . meta_classifier : object The meta-classifier to be fitted on the ensemble of classifiers use_probas : bool (default: False) If True, trains meta-classifier based on predicted probabilities instead of class labels. drop_proba_col : string (default: None) Drops extra \"probability\" column in the feature set, because it is redundant: p(y_c) = 1 - p(y_1) + p(y_2) + ... + p(y_{c-1}). This can be useful for meta-classifiers that are sensitive to perfectly collinear features. If 'last', drops last probability column. If 'first', drops first probability column. Only relevant if use_probas=True . cv : int, cross-validation generator or an iterable, optional (default: 2) Determines the cross-validation splitting strategy. Possible inputs for cv are: - None, to use the default 2-fold cross validation, - integer, to specify the number of folds in a (Stratified)KFold , - An object to be used as a cross-validation generator. - An iterable yielding train, test splits. For integer/None inputs, it will use either a KFold or StratifiedKFold cross validation depending the value of stratify argument. shuffle : bool (default: True) If True, and the cv argument is integer, the training data will be shuffled at fitting stage prior to cross-validation. If the cv argument is a specific cross validation technique, this argument is omitted. random_state : int, RandomState instance or None, optional (default: None) Constrols the randomness of the cv splitter. Used when cv is integer and shuffle=True . New in v0.16.0. stratify : bool (default: True) If True, and the cv argument is integer it will follow a stratified K-Fold cross validation technique. If the cv argument is a specific cross validation technique, this argument is omitted. verbose : int, optional (default=0) Controls the verbosity of the building process. - verbose=0 (default): Prints nothing - verbose=1 : Prints the number & name of the regressor being fitted and which fold is currently being used for fitting - verbose=2 : Prints info about the parameters of the regressor being fitted - verbose>2 : Changes verbose param of the underlying regressor to self.verbose - 2 use_features_in_secondary : bool (default: False) If True, the meta-classifier will be trained both on the predictions of the original classifiers and the original dataset. If False, the meta-classifier will be trained only on the predictions of the original classifiers. store_train_meta_features : bool (default: False) If True, the meta-features computed from the training data used for fitting the meta-classifier stored in the self.train_meta_features_ array, which can be accessed after calling fit . use_clones : bool (default: True) Clones the classifiers for stacking classification if True (default) or else uses the original ones, which will be refitted on the dataset upon calling the fit method. Hence, if use_clones=True, the original input classifiers will remain unmodified upon using the StackingCVClassifier's fit method. Setting use_clones=False is recommended if you are working with estimators that are supporting the scikit-learn fit/predict API interface but are not compatible to scikit-learn's clone function. n_jobs : int or None, optional (default=None) The number of CPUs to use to do the computation. None means 1 unless in a joblib.parallel_backend context. -1 means using all processors. for more details. New in v0.16.0. pre_dispatch : int, or string, optional Controls the number of jobs that get dispatched during parallel execution. Reducing this number can be useful to avoid an explosion of memory consumption when more jobs get dispatched than CPUs can process. This parameter can be: - None, in which case all the jobs are immediately created and spawned. Use this for lightweight and fast-running jobs, to avoid delays due to on-demand spawning of the jobs - An int, giving the exact number of total jobs that are spawned - A string, giving an expression as a function of n_jobs, as in '2*n_jobs' New in v0.16.0. Attributes clfs_ : list, shape=[n_classifiers] Fitted classifiers (clones of the original classifiers) meta_clf_ : estimator Fitted meta-classifier (clone of the original meta-estimator) train_meta_features : numpy array, shape = [n_samples, n_classifiers] meta-features for training data, where n_samples is the number of samples in training data and n_classifiers is the number of classfiers. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/classifier/StackingCVClassifier/","title":"StackingCVClassifier"},{"location":"api_subpackages/mlxtend.classifier/#methods_7","text":"decision_function(X) Predict class confidence scores for X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns scores : shape=(n_samples,) if n_classes == 2 else (n_samples, n_classes). Confidence scores per (sample, class) combination. In the binary case, confidence score for self.classes_[1] where >0 means this class would be predicted. fit(X, y, groups=None, sample_weight=None) Fit ensemble classifers and the meta-classifier. Parameters X : numpy array, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : numpy array, shape = [n_samples] Target values. groups : numpy array/None, shape = [n_samples] The group that each sample belongs to. This is used by specific folding strategies such as GroupKFold() sample_weight : array-like, shape = [n_samples], optional Sample weights passed as sample_weights to each regressor in the regressors list as well as the meta_regressor. Raises error if some regressor does not support sample_weight in the fit() method. Returns self : object fit_transform(X, y=None, fit_params) Fit to data, then transform it. Fits transformer to `X` and `y` with optional parameters `fit_params` and returns a transformed version of `X`. Parameters X : array-like of shape (n_samples, n_features) Input samples. y : array-like of shape (n_samples,) or (n_samples, n_outputs), default=None Target values (None for unsupervised transformations). **fit_params : dict Additional fit parameters. Returns X_new : ndarray array of shape (n_samples, n_features_new) Transformed array. get_params(deep=True) Return estimator parameter names for GridSearch support. predict(X) Predict target values for X. Parameters X : numpy array, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns labels : array-like, shape = [n_samples] Predicted class labels. predict_meta_features(X) Get meta-features of test-data. Parameters X : numpy array, shape = [n_samples, n_features] Test vectors, where n_samples is the number of samples and n_features is the number of features. Returns meta-features : numpy array, shape = [n_samples, n_classifiers] Returns the meta-features for test data. predict_proba(X) Predict class probabilities for X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns proba : array-like, shape = [n_samples, n_classes] or a list of n_outputs of such arrays if n_outputs > 1. Probability for each class per sample. score(X, y, sample_weight=None) Return the mean accuracy on the given test data and labels. In multi-label classification, this is the subset accuracy which is a harsh metric since you require for each sample that each label set be correctly predicted. Parameters X : array-like of shape (n_samples, n_features) Test samples. y : array-like of shape (n_samples,) or (n_samples, n_outputs) True labels for X . sample_weight : array-like of shape (n_samples,), default=None Sample weights. Returns score : float Mean accuracy of self.predict(X) w.r.t. y . set_output( , transform=None)* Set output container. See :ref:`sphx_glr_auto_examples_miscellaneous_plot_set_output.py` for an example on how to use the API. Parameters transform : {\"default\", \"pandas\"}, default=None Configure output of transform and fit_transform . \"default\" : Default output format of a transformer \"pandas\" : DataFrame output None : Transform configuration is unchanged Returns self : estimator instance Estimator instance. set_params( params) Set the parameters of this estimator. Valid parameter keys can be listed with ``get_params()``. Returns self","title":"Methods"},{"location":"api_subpackages/mlxtend.classifier/#properties","text":"named_classifiers None","title":"Properties"},{"location":"api_subpackages/mlxtend.classifier/#stackingclassifier","text":"StackingClassifier(classifiers, meta_classifier, use_probas=False, drop_proba_col=None, average_probas=False, verbose=0, use_features_in_secondary=False, store_train_meta_features=False, use_clones=True, fit_base_estimators=True) A Stacking classifier for scikit-learn estimators for classification. Parameters classifiers : array-like, shape = [n_classifiers] A list of classifiers. Invoking the fit method on the StackingClassifer will fit clones of these original classifiers that will be stored in the class attribute self.clfs_ if use_clones=True (default) and fit_base_estimators=True (default). meta_classifier : object The meta-classifier to be fitted on the ensemble of classifiers use_probas : bool (default: False) If True, trains meta-classifier based on predicted probabilities instead of class labels. drop_proba_col : string (default: None) Drops extra \"probability\" column in the feature set, because it is redundant: p(y_c) = 1 - p(y_1) + p(y_2) + ... + p(y_{c-1}). This can be useful for meta-classifiers that are sensitive to perfectly collinear features. If 'last', drops last probability column. If 'first', drops first probability column. Only relevant if use_probas=True . average_probas : bool (default: False) Averages the probabilities as meta features if True . Only relevant if use_probas=True . verbose : int, optional (default=0) Controls the verbosity of the building process. - verbose=0 (default): Prints nothing - verbose=1 : Prints the number & name of the regressor being fitted - verbose=2 : Prints info about the parameters of the regressor being fitted - verbose>2 : Changes verbose param of the underlying regressor to self.verbose - 2 use_features_in_secondary : bool (default: False) If True, the meta-classifier will be trained both on the predictions of the original classifiers and the original dataset. If False, the meta-classifier will be trained only on the predictions of the original classifiers. store_train_meta_features : bool (default: False) If True, the meta-features computed from the training data used for fitting the meta-classifier stored in the self.train_meta_features_ array, which can be accessed after calling fit . use_clones : bool (default: True) Clones the classifiers for stacking classification if True (default) or else uses the original ones, which will be refitted on the dataset upon calling the fit method. Hence, if use_clones=True, the original input classifiers will remain unmodified upon using the StackingClassifier's fit method. Setting use_clones=False is recommended if you are working with estimators that are supporting the scikit-learn fit/predict API interface but are not compatible to scikit-learn's clone function. fit_base_estimators: bool (default: True) Refits classifiers in classifiers if True; uses references to the classifiers , otherwise (assumes that the classifiers were already fit). Note: fit_base_estimators=False will enforce use_clones to be False, and is incompatible to most scikit-learn wrappers! For instance, if any form of cross-validation is performed this would require the re-fitting classifiers to training folds, which would raise a NotFitterError if fit_base_estimators=False. (New in mlxtend v0.6.) Attributes clfs_ : list, shape=[n_classifiers] Fitted classifiers (clones of the original classifiers) meta_clf_ : estimator Fitted meta-classifier (clone of the original meta-estimator) train_meta_features : numpy array, shape = [n_samples, n_classifiers] meta-features for training data, where n_samples is the number of samples in training data and n_classifiers is the number of classfiers. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/classifier/StackingClassifier/","title":"StackingClassifier"},{"location":"api_subpackages/mlxtend.classifier/#methods_8","text":"decision_function(X) Predict class confidence scores for X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns scores : shape=(n_samples,) if n_classes == 2 else (n_samples, n_classes). Confidence scores per (sample, class) combination. In the binary case, confidence score for self.classes_[1] where >0 means this class would be predicted. fit(X, y, sample_weight=None) Fit ensemble classifers and the meta-classifier. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] or [n_samples, n_outputs] Target values. sample_weight : array-like, shape = [n_samples], optional Sample weights passed as sample_weights to each regressor in the regressors list as well as the meta_regressor. Raises error if some regressor does not support sample_weight in the fit() method. Returns self : object fit_transform(X, y=None, fit_params) Fit to data, then transform it. Fits transformer to `X` and `y` with optional parameters `fit_params` and returns a transformed version of `X`. Parameters X : array-like of shape (n_samples, n_features) Input samples. y : array-like of shape (n_samples,) or (n_samples, n_outputs), default=None Target values (None for unsupervised transformations). **fit_params : dict Additional fit parameters. Returns X_new : ndarray array of shape (n_samples, n_features_new) Transformed array. get_params(deep=True) Return estimator parameter names for GridSearch support. predict(X) Predict target values for X. Parameters X : numpy array, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns labels : array-like, shape = [n_samples] Predicted class labels. predict_meta_features(X) Get meta-features of test-data. Parameters X : numpy array, shape = [n_samples, n_features] Test vectors, where n_samples is the number of samples and n_features is the number of features. Returns meta-features : numpy array, shape = [n_samples, n_classifiers] Returns the meta-features for test data. predict_proba(X) Predict class probabilities for X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns proba : array-like, shape = [n_samples, n_classes] or a list of n_outputs of such arrays if n_outputs > 1. Probability for each class per sample. score(X, y, sample_weight=None) Return the mean accuracy on the given test data and labels. In multi-label classification, this is the subset accuracy which is a harsh metric since you require for each sample that each label set be correctly predicted. Parameters X : array-like of shape (n_samples, n_features) Test samples. y : array-like of shape (n_samples,) or (n_samples, n_outputs) True labels for X . sample_weight : array-like of shape (n_samples,), default=None Sample weights. Returns score : float Mean accuracy of self.predict(X) w.r.t. y . set_output( , transform=None)* Set output container. See :ref:`sphx_glr_auto_examples_miscellaneous_plot_set_output.py` for an example on how to use the API. Parameters transform : {\"default\", \"pandas\"}, default=None Configure output of transform and fit_transform . \"default\" : Default output format of a transformer \"pandas\" : DataFrame output None : Transform configuration is unchanged Returns self : estimator instance Estimator instance. set_params( params) Set the parameters of this estimator. Valid parameter keys can be listed with ``get_params()``. Returns self","title":"Methods"},{"location":"api_subpackages/mlxtend.classifier/#properties_1","text":"named_classifiers None","title":"Properties"},{"location":"api_subpackages/mlxtend.cluster/","text":"mlxtend version: 0.23.1 Kmeans Kmeans(k, max_iter=10, convergence_tolerance=1e-05, random_seed=None, print_progress=0) K-means clustering class. Added in 0.4.1dev Parameters k : int Number of clusters max_iter : int (default: 10) Number of iterations during cluster assignment. Cluster re-assignment stops automatically when the algorithm converged. convergence_tolerance : float (default: 1e-05) Compares current centroids with centroids of the previous iteration using the given tolerance (a small positive float)to determine if the algorithm converged early. random_seed : int (default: None) Set random state for the initial centroid assignment. print_progress : int (default: 0) Prints progress in fitting to stderr. 0: No output 1: Iterations elapsed 2: 1 plus time elapsed 3: 2 plus estimated time until completion Attributes centroids_ : 2d-array, shape={k, n_features} Feature values of the k cluster centroids. custers_ : dictionary The cluster assignments stored as a Python dictionary; the dictionary keys denote the cluster indeces and the items are Python lists of the sample indices that were assigned to each cluster. iterations_ : int Number of iterations until convergence. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/classifier/Kmeans/ Methods fit(X, init_params=True) Learn model from training data. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. init_params : bool (default: True) Re-initializes model parameters prior to fitting. Set False to continue training with weights from a previous model fitting. Returns self : object get_params(deep=True) Get parameters for this estimator. Parameters deep : boolean, optional If True, will return the parameters for this estimator and contained subobjects that are estimators. Returns params : mapping of string to any Parameter names mapped to their values.' adapted from https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/base.py Author: Gael Varoquaux gael.varoquaux@normalesup.org License: BSD 3 clause predict(X) Predict targets from X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns target_values : array-like, shape = [n_samples] Predicted target values. set_params( params) Set the parameters of this estimator. The method works on simple estimators as well as on nested objects (such as pipelines). The latter have parameters of the form <component>__<parameter> so that it's possible to update each component of a nested object. Returns self adapted from https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/base.py Author: Gael Varoquaux <gael.varoquaux@normalesup.org> License: BSD 3 clause","title":"Mlxtend.cluster"},{"location":"api_subpackages/mlxtend.cluster/#kmeans","text":"Kmeans(k, max_iter=10, convergence_tolerance=1e-05, random_seed=None, print_progress=0) K-means clustering class. Added in 0.4.1dev Parameters k : int Number of clusters max_iter : int (default: 10) Number of iterations during cluster assignment. Cluster re-assignment stops automatically when the algorithm converged. convergence_tolerance : float (default: 1e-05) Compares current centroids with centroids of the previous iteration using the given tolerance (a small positive float)to determine if the algorithm converged early. random_seed : int (default: None) Set random state for the initial centroid assignment. print_progress : int (default: 0) Prints progress in fitting to stderr. 0: No output 1: Iterations elapsed 2: 1 plus time elapsed 3: 2 plus estimated time until completion Attributes centroids_ : 2d-array, shape={k, n_features} Feature values of the k cluster centroids. custers_ : dictionary The cluster assignments stored as a Python dictionary; the dictionary keys denote the cluster indeces and the items are Python lists of the sample indices that were assigned to each cluster. iterations_ : int Number of iterations until convergence. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/classifier/Kmeans/","title":"Kmeans"},{"location":"api_subpackages/mlxtend.cluster/#methods","text":"fit(X, init_params=True) Learn model from training data. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. init_params : bool (default: True) Re-initializes model parameters prior to fitting. Set False to continue training with weights from a previous model fitting. Returns self : object get_params(deep=True) Get parameters for this estimator. Parameters deep : boolean, optional If True, will return the parameters for this estimator and contained subobjects that are estimators. Returns params : mapping of string to any Parameter names mapped to their values.' adapted from https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/base.py Author: Gael Varoquaux gael.varoquaux@normalesup.org License: BSD 3 clause predict(X) Predict targets from X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns target_values : array-like, shape = [n_samples] Predicted target values. set_params( params) Set the parameters of this estimator. The method works on simple estimators as well as on nested objects (such as pipelines). The latter have parameters of the form <component>__<parameter> so that it's possible to update each component of a nested object. Returns self adapted from https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/base.py Author: Gael Varoquaux <gael.varoquaux@normalesup.org> License: BSD 3 clause","title":"Methods"},{"location":"api_subpackages/mlxtend.data/","text":"mlxtend version: 0.23.1 autompg_data autompg_data() Auto MPG dataset. Source : https://archive.ics.uci.edu/ml/datasets/Auto+MPG Number of samples : 392 Continuous target variable : mpg Dataset Attributes: 1) cylinders: multi-valued discrete 2) displacement: continuous 3) horsepower: continuous 4) weight: continuous 5) acceleration: continuous 6) model year: multi-valued discrete 7) origin: multi-valued discrete 8) car name: string (unique for each instance) Returns X, y : [n_samples, n_features], [n_targets] X is the feature matrix with 392 auto samples as rows and 8 feature columns (6 rows with NaNs removed). y is a 1-dimensional array of the target MPG values. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/data/autompg_data/ boston_housing_data boston_housing_data() Boston Housing dataset. Source : https://archive.ics.uci.edu/ml/datasets/Housing Number of samples : 506 Continuous target variable : MEDV MEDV = Median value of owner-occupied homes in $1000's Dataset Attributes: 1) CRIM per capita crime rate by town 2) ZN proportion of residential land zoned for lots over 25,000 sq.ft. 3) INDUS proportion of non-retail business acres per town 4) CHAS Charles River dummy variable (= 1 if tract bounds river; 0 otherwise) 5) NOX nitric oxides concentration (parts per 10 million) 6) RM average number of rooms per dwelling 7) AGE proportion of owner-occupied units built prior to 1940 8) DIS weighted distances to five Boston employment centres 9) RAD index of accessibility to radial highways 10) TAX full-value property-tax rate per $10,000 11) PTRATIO pupil-teacher ratio by town 12) B 1000(Bk - 0.63)^2 where Bk is the prop. of b. by town 13) LSTAT % lower status of the population Returns X, y : [n_samples, n_features], [n_class_labels] X is the feature matrix with 506 housing samples as rows and 13 feature columns. y is a 1-dimensional array of the continuous target variable MEDV Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/data/boston_housing_data/ iris_data iris_data(version='uci') Iris flower dataset. Source : https://archive.ics.uci.edu/ml/datasets/Iris Number of samples : 150 Class labels : {0, 1, 2}, distribution: [50, 50, 50] 0 = setosa, 1 = versicolor, 2 = virginica. Dataset Attributes: 1) sepal length [cm] 2) sepal width [cm] 3) petal length [cm] 4) petal width [cm] Parameters version : string, optional (default: 'uci'). Version to use {'uci', 'corrected'}. 'uci' loads the dataset as deposited on the UCI machine learning repository, and 'corrected' provides the version that is consistent with Fisher's original paper. See Note for details. Returns X, y : [n_samples, n_features], [n_class_labels] X is the feature matrix with 150 flower samples as rows, and 4 feature columns sepal length, sepal width, petal length, and petal width. y is a 1-dimensional array of the class labels {0, 1, 2} Note The Iris dataset (originally collected by Edgar Anderson) and available in UCI's machine learning repository is different from the Iris dataset described in the original paper by R.A. Fisher [1]). Precisely, there are two data points (row number 34 and 37) in UCI's Machine Learning repository are different from the origianlly published Iris dataset. Also, the original version of the Iris Dataset, which can be loaded via version='corrected' is the same as the one in R. [1] . A. Fisher (1936). \"The use of multiple measurements in taxonomic problems\". Annals of Eugenics. 7 (2): 179\u2013188 Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/data/iris_data/ loadlocal_mnist loadlocal_mnist(images_path, labels_path) Read MNIST from ubyte files. Parameters images_path : str path to the test or train MNIST ubyte file labels_path : str path to the test or train MNIST class labels file Returns images : [n_samples, n_pixels] numpy.array Pixel values of the images. labels : [n_samples] numpy array Target class labels Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/data/loadlocal_mnist/ make_multiplexer_dataset make_multiplexer_dataset(address_bits=2, sample_size=100, positive_class_ratio=0.5, shuffle=False, random_seed=None) Function to create a binary n-bit multiplexer dataset. New in mlxtend v0.9 Parameters address_bits : int (default: 2) A positive integer that determines the number of address bits in the multiplexer, which in turn determine the n-bit capacity of the multiplexer and therefore the number of features. The number of features is determined by the number of address bits. For example, 2 address bits will result in a 6 bit multiplexer and consequently 6 features (2 + 2^2 = 6). If address_bits=3 , then this results in an 11-bit multiplexer as (2 + 2^3 = 11) with 11 features. sample_size : int (default: 100) The total number of samples generated. positive_class_ratio : float (default: 0.5) The fraction (a float between 0 and 1) of samples in the sample_size d dataset that have class label 1. If positive_class_ratio=0.5 (default), then the ratio of class 0 and class 1 samples is perfectly balanced. shuffle : Bool (default: False) Whether or not to shuffle the features and labels. If False (default), the samples are returned in sorted order starting with sample_size /2 samples with class label 0 and followed by sample_size /2 samples with class label 1. random_seed : int (default: None) Random seed used for generating the multiplexer samples and shuffling. Returns X, y : [n_samples, n_features], [n_class_labels] X is the feature matrix with the number of samples equal to sample_size . The number of features is determined by the number of address bits. For instance, 2 address bits will result in a 6 bit multiplexer and consequently 6 features (2 + 2^2 = 6). All features are binary (values in {0, 1}). y is a 1-dimensional array of class labels in {0, 1}. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/data/make_multiplexer_dataset mnist_data mnist_data() 5000 samples from the MNIST handwritten digits dataset. Data Source : https://yann.lecun.com/exdb/mnist/ Returns X, y : [n_samples, n_features], [n_class_labels] X is the feature matrix with 5000 image samples as rows, each row consists of 28x28 pixels that were unrolled into 784 pixel feature vectors. y contains the 10 unique class labels 0-9. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/data/mnist_data/ three_blobs_data three_blobs_data() A random dataset of 3 2D blobs for clustering. Number of samples : 150 Suggested labels : {0, 1, 2}, distribution: [50, 50, 50] Returns X, y : [n_samples, n_features], [n_cluster_labels] X is the feature matrix with 159 samples as rows and 2 feature columns. y is a 1-dimensional array of the 3 suggested cluster labels 0, 1, 2 Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/data/three_blobs_data wine_data wine_data() Wine dataset. Source : https://archive.ics.uci.edu/ml/datasets/Wine Number of samples : 178 Class labels : {0, 1, 2}, distribution: [59, 71, 48] Dataset Attributes: 1) Alcohol 2) Malic acid 3) Ash 4) Alcalinity of ash 5) Magnesium 6) Total phenols 7) Flavanoids 8) Nonflavanoid phenols 9) Proanthocyanins 10) Color intensity 11) Hue 12) OD280/OD315 of diluted wines 13) Proline Returns X, y : [n_samples, n_features], [n_class_labels] X is the feature matrix with 178 wine samples as rows and 13 feature columns. y is a 1-dimensional array of the 3 class labels 0, 1, 2 Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/data/wine_data","title":"Mlxtend.data"},{"location":"api_subpackages/mlxtend.data/#autompg_data","text":"autompg_data() Auto MPG dataset. Source : https://archive.ics.uci.edu/ml/datasets/Auto+MPG Number of samples : 392 Continuous target variable : mpg Dataset Attributes: 1) cylinders: multi-valued discrete 2) displacement: continuous 3) horsepower: continuous 4) weight: continuous 5) acceleration: continuous 6) model year: multi-valued discrete 7) origin: multi-valued discrete 8) car name: string (unique for each instance) Returns X, y : [n_samples, n_features], [n_targets] X is the feature matrix with 392 auto samples as rows and 8 feature columns (6 rows with NaNs removed). y is a 1-dimensional array of the target MPG values. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/data/autompg_data/","title":"autompg_data"},{"location":"api_subpackages/mlxtend.data/#boston_housing_data","text":"boston_housing_data() Boston Housing dataset. Source : https://archive.ics.uci.edu/ml/datasets/Housing Number of samples : 506 Continuous target variable : MEDV MEDV = Median value of owner-occupied homes in $1000's Dataset Attributes: 1) CRIM per capita crime rate by town 2) ZN proportion of residential land zoned for lots over 25,000 sq.ft. 3) INDUS proportion of non-retail business acres per town 4) CHAS Charles River dummy variable (= 1 if tract bounds river; 0 otherwise) 5) NOX nitric oxides concentration (parts per 10 million) 6) RM average number of rooms per dwelling 7) AGE proportion of owner-occupied units built prior to 1940 8) DIS weighted distances to five Boston employment centres 9) RAD index of accessibility to radial highways 10) TAX full-value property-tax rate per $10,000 11) PTRATIO pupil-teacher ratio by town 12) B 1000(Bk - 0.63)^2 where Bk is the prop. of b. by town 13) LSTAT % lower status of the population Returns X, y : [n_samples, n_features], [n_class_labels] X is the feature matrix with 506 housing samples as rows and 13 feature columns. y is a 1-dimensional array of the continuous target variable MEDV Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/data/boston_housing_data/","title":"boston_housing_data"},{"location":"api_subpackages/mlxtend.data/#iris_data","text":"iris_data(version='uci') Iris flower dataset. Source : https://archive.ics.uci.edu/ml/datasets/Iris Number of samples : 150 Class labels : {0, 1, 2}, distribution: [50, 50, 50] 0 = setosa, 1 = versicolor, 2 = virginica. Dataset Attributes: 1) sepal length [cm] 2) sepal width [cm] 3) petal length [cm] 4) petal width [cm] Parameters version : string, optional (default: 'uci'). Version to use {'uci', 'corrected'}. 'uci' loads the dataset as deposited on the UCI machine learning repository, and 'corrected' provides the version that is consistent with Fisher's original paper. See Note for details. Returns X, y : [n_samples, n_features], [n_class_labels] X is the feature matrix with 150 flower samples as rows, and 4 feature columns sepal length, sepal width, petal length, and petal width. y is a 1-dimensional array of the class labels {0, 1, 2} Note The Iris dataset (originally collected by Edgar Anderson) and available in UCI's machine learning repository is different from the Iris dataset described in the original paper by R.A. Fisher [1]). Precisely, there are two data points (row number 34 and 37) in UCI's Machine Learning repository are different from the origianlly published Iris dataset. Also, the original version of the Iris Dataset, which can be loaded via version='corrected' is the same as the one in R. [1] . A. Fisher (1936). \"The use of multiple measurements in taxonomic problems\". Annals of Eugenics. 7 (2): 179\u2013188 Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/data/iris_data/","title":"iris_data"},{"location":"api_subpackages/mlxtend.data/#loadlocal_mnist","text":"loadlocal_mnist(images_path, labels_path) Read MNIST from ubyte files. Parameters images_path : str path to the test or train MNIST ubyte file labels_path : str path to the test or train MNIST class labels file Returns images : [n_samples, n_pixels] numpy.array Pixel values of the images. labels : [n_samples] numpy array Target class labels Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/data/loadlocal_mnist/","title":"loadlocal_mnist"},{"location":"api_subpackages/mlxtend.data/#make_multiplexer_dataset","text":"make_multiplexer_dataset(address_bits=2, sample_size=100, positive_class_ratio=0.5, shuffle=False, random_seed=None) Function to create a binary n-bit multiplexer dataset. New in mlxtend v0.9 Parameters address_bits : int (default: 2) A positive integer that determines the number of address bits in the multiplexer, which in turn determine the n-bit capacity of the multiplexer and therefore the number of features. The number of features is determined by the number of address bits. For example, 2 address bits will result in a 6 bit multiplexer and consequently 6 features (2 + 2^2 = 6). If address_bits=3 , then this results in an 11-bit multiplexer as (2 + 2^3 = 11) with 11 features. sample_size : int (default: 100) The total number of samples generated. positive_class_ratio : float (default: 0.5) The fraction (a float between 0 and 1) of samples in the sample_size d dataset that have class label 1. If positive_class_ratio=0.5 (default), then the ratio of class 0 and class 1 samples is perfectly balanced. shuffle : Bool (default: False) Whether or not to shuffle the features and labels. If False (default), the samples are returned in sorted order starting with sample_size /2 samples with class label 0 and followed by sample_size /2 samples with class label 1. random_seed : int (default: None) Random seed used for generating the multiplexer samples and shuffling. Returns X, y : [n_samples, n_features], [n_class_labels] X is the feature matrix with the number of samples equal to sample_size . The number of features is determined by the number of address bits. For instance, 2 address bits will result in a 6 bit multiplexer and consequently 6 features (2 + 2^2 = 6). All features are binary (values in {0, 1}). y is a 1-dimensional array of class labels in {0, 1}. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/data/make_multiplexer_dataset","title":"make_multiplexer_dataset"},{"location":"api_subpackages/mlxtend.data/#mnist_data","text":"mnist_data() 5000 samples from the MNIST handwritten digits dataset. Data Source : https://yann.lecun.com/exdb/mnist/ Returns X, y : [n_samples, n_features], [n_class_labels] X is the feature matrix with 5000 image samples as rows, each row consists of 28x28 pixels that were unrolled into 784 pixel feature vectors. y contains the 10 unique class labels 0-9. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/data/mnist_data/","title":"mnist_data"},{"location":"api_subpackages/mlxtend.data/#three_blobs_data","text":"three_blobs_data() A random dataset of 3 2D blobs for clustering. Number of samples : 150 Suggested labels : {0, 1, 2}, distribution: [50, 50, 50] Returns X, y : [n_samples, n_features], [n_cluster_labels] X is the feature matrix with 159 samples as rows and 2 feature columns. y is a 1-dimensional array of the 3 suggested cluster labels 0, 1, 2 Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/data/three_blobs_data","title":"three_blobs_data"},{"location":"api_subpackages/mlxtend.data/#wine_data","text":"wine_data() Wine dataset. Source : https://archive.ics.uci.edu/ml/datasets/Wine Number of samples : 178 Class labels : {0, 1, 2}, distribution: [59, 71, 48] Dataset Attributes: 1) Alcohol 2) Malic acid 3) Ash 4) Alcalinity of ash 5) Magnesium 6) Total phenols 7) Flavanoids 8) Nonflavanoid phenols 9) Proanthocyanins 10) Color intensity 11) Hue 12) OD280/OD315 of diluted wines 13) Proline Returns X, y : [n_samples, n_features], [n_class_labels] X is the feature matrix with 178 wine samples as rows and 13 feature columns. y is a 1-dimensional array of the 3 class labels 0, 1, 2 Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/data/wine_data","title":"wine_data"},{"location":"api_subpackages/mlxtend.evaluate/","text":"mlxtend version: 0.23.1 BootstrapOutOfBag BootstrapOutOfBag(n_splits=200, random_seed=None) Parameters n_splits : int (default=200) Number of bootstrap iterations. Must be larger than 1. random_seed : int (default=None) If int, random_seed is the seed used by the random number generator. Returns train_idx : ndarray The training set indices for that split. test_idx : ndarray The testing set indices for that split. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/BootstrapOutOfBag/ Methods get_n_splits(X=None, y=None, groups=None) Returns the number of splitting iterations in the cross-validator Parameters X : object Always ignored, exists for compatibility with scikit-learn. y : object Always ignored, exists for compatibility with scikit-learn. groups : object Always ignored, exists for compatibility with scikit-learn. Returns n_splits : int Returns the number of splitting iterations in the cross-validator. split(X, y=None, groups=None) y : array-like or None (default: None) Argument is not used and only included as parameter for compatibility, similar to KFold in scikit-learn. groups : array-like or None (default: None) Argument is not used and only included as parameter for compatibility, similar to KFold in scikit-learn. GroupTimeSeriesSplit GroupTimeSeriesSplit(test_size, train_size=None, n_splits=None, gap_size=0, shift_size=1, window_type='rolling') Group time series cross-validator. Parameters test_size : int Size of test dataset. train_size : int (default=None) Size of train dataset. n_splits : int (default=None) Number of the splits. gap_size : int (default=0) Gap size between train and test datasets. shift_size : int (default=1) Step to shift for the next fold. window_type : str (default=\"rolling\") Type of the window. Possible values: \"rolling\", \"expanding\". Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/GroupTimeSeriesSplit/ Methods get_n_splits(X=None, y=None, groups=None) Returns the number of splitting iterations in the cross-validator. Parameters X : object Always ignored, exists for compatibility. y : object Always ignored, exists for compatibility. groups : object Always ignored, exists for compatibility. Returns n_splits : int Returns the number of splitting iterations in the cross-validator. split(X, y=None, groups=None) Generate indices to split data into training and test set. Parameters X : array-like Training data. y : array-like (default=None) Always ignored, exists for compatibility. groups : array-like (default=None) Array with group names or sequence numbers. Yields train : ndarray The training set indices for that split. test : ndarray The testing set indices for that split. PredefinedHoldoutSplit PredefinedHoldoutSplit(valid_indices) Train/Validation set splitter for sklearn's GridSearchCV etc. Uses user-specified train/validation set indices to split a dataset into train/validation sets using user-defined or random indices. Parameters valid_indices : array-like, shape (num_examples,) Indices of the training examples in the training set to be used for validation. All other indices in the training set are used to for a training subset for model fitting. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/PredefinedHoldoutSplit/ Methods get_n_splits(X=None, y=None, groups=None) Returns the number of splitting iterations in the cross-validator Parameters X : object Always ignored, exists for compatibility. y : object Always ignored, exists for compatibility. groups : object Always ignored, exists for compatibility. Returns n_splits : 1 Returns the number of splitting iterations in the cross-validator. Always returns 1. split(X, y, groups=None) Generate indices to split data into training and test set. Parameters X : array-like, shape (num_examples, num_features) Training data, where num_examples is the number of examples and num_features is the number of features. y : array-like, shape (num_examples,) The target variable for supervised learning problems. Stratification is done based on the y labels. groups : object Always ignored, exists for compatibility. Yields train_index : ndarray The training set indices for that split. valid_index : ndarray The validation set indices for that split. RandomHoldoutSplit RandomHoldoutSplit(valid_size=0.5, random_seed=None, stratify=False) Train/Validation set splitter for sklearn's GridSearchCV etc. Provides train/validation set indices to split a dataset into train/validation sets using random indices. Parameters valid_size : float (default: 0.5) Proportion of examples that being assigned as validation examples. 1- valid_size will then automatically be assigned as training set examples. random_seed : int (default: None) The random seed for splitting the data into training and validation set partitions. stratify : bool (default: False) True or False, whether to perform a stratified split or not Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/RandomHoldoutSplit/ Methods get_n_splits(X=None, y=None, groups=None) Returns the number of splitting iterations in the cross-validator Parameters X : object Always ignored, exists for compatibility. y : object Always ignored, exists for compatibility. groups : object Always ignored, exists for compatibility. Returns n_splits : 1 Returns the number of splitting iterations in the cross-validator. Always returns 1. split(X, y, groups=None) Generate indices to split data into training and test set. Parameters X : array-like, shape (num_examples, num_features) Training data, where num_examples is the number of training examples and num_features is the number of features. y : array-like, shape (num_examples,) The target variable for supervised learning problems. Stratification is done based on the y labels. groups : object Always ignored, exists for compatibility. Yields train_index : ndarray The training set indices for that split. valid_index : ndarray The validation set indices for that split. accuracy_score accuracy_score(y_target, y_predicted, method='standard', pos_label=1, normalize=True) General accuracy function for supervised learning. Parameters y_target : array-like, shape=[n_values] True class labels or target values. y_predicted : array-like, shape=[n_values] Predicted class labels or target values. method : str, 'standard' by default. The chosen method for accuracy computation. If set to 'standard', computes overall accuracy. If set to 'binary', computes accuracy for class pos_label. If set to 'average', computes average per-class (balanced) accuracy. If set to 'balanced', computes the scikit-learn-style balanced accuracy. pos_label : str or int, 1 by default. The class whose accuracy score is to be reported. Used only when method is set to 'binary' normalize : bool, True by default. If True, returns fraction of correctly classified samples. If False, returns number of correctly classified samples. Returns score: float Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/accuracy_score/ bias_variance_decomp bias_variance_decomp(estimator, X_train, y_train, X_test, y_test, loss='0-1_loss', num_rounds=200, random_seed=None, fit_params) estimator : object A classifier or regressor object or class implementing both a fit and predict method similar to the scikit-learn API. X_train : array-like, shape=(num_examples, num_features) A training dataset for drawing the bootstrap samples to carry out the bias-variance decomposition. y_train : array-like, shape=(num_examples) Targets (class labels, continuous values in case of regression) associated with the X_train examples. X_test : array-like, shape=(num_examples, num_features) The test dataset for computing the average loss, bias, and variance. y_test : array-like, shape=(num_examples) Targets (class labels, continuous values in case of regression) associated with the X_test examples. loss : str (default='0-1_loss') Loss function for performing the bias-variance decomposition. Currently allowed values are '0-1_loss' and 'mse'. num_rounds : int (default=200) Number of bootstrap rounds (sampling from the training set) for performing the bias-variance decomposition. Each bootstrap sample has the same size as the original training set. random_seed : int (default=None) Random seed for the bootstrap sampling used for the bias-variance decomposition. fit_params : additional parameters Additional parameters to be passed to the .fit() function of the estimator when it is fit to the bootstrap samples. Returns avg_expected_loss, avg_bias, avg_var : returns the average expected average bias, and average bias (all floats), where the average is computed over the data points in the test set. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/bias_variance_decomp/ bootstrap bootstrap(x, func, num_rounds=1000, ci=0.95, ddof=1, seed=None) Implements the ordinary nonparametric bootstrap Parameters x : NumPy array, shape=(n_samples, [n_columns]) An one or multidimensional array of data records func : A function which computes a statistic that is used to compute the bootstrap replicates (the statistic computed from the bootstrap samples). This function must return a scalar value. For example, np.mean or np.median would be an acceptable argument for func if x is a 1-dimensional array or vector. num_rounds : int (default=1000) The number of bootstrap samples to draw where each bootstrap sample has the same number of records as the original dataset. ci : int (default=0.95) An integer in the range (0, 1) that represents the confidence level for computing the confidence interval. For example, ci=0.95 (default) will compute the 95% confidence interval from the bootstrap replicates. ddof : int The delta degrees of freedom used when computing the standard error. seed : int or None (default=None) Random seed for generating bootstrap samples. Returns original, standard_error, (lower_ci, upper_ci) : tuple Returns the statistic of the original sample ( original ), the standard error of the estimate, and the respective confidence interval bounds. Examples ``` >>> from mlxtend.evaluate import bootstrap >>> rng = np.random.RandomState(123) >>> x = rng.normal(loc=5., size=100) >>> original, std_err, ci_bounds = bootstrap(x, ... num_rounds=1000, ... func=np.mean, ... ci=0.95, ... seed=123) >>> print('Mean: %.2f, SE: +/- %.2f, CI95: [%.2f, %.2f]' % (original, ... std_err, ... ci_bounds[0], ... ci_bounds[1])) Mean: 5.03, SE: +/- 0.11, CI95: [4.80, 5.26] >>> For more usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/bootstrap/ ## bootstrap_point632_score *bootstrap_point632_score(estimator, X, y, n_splits=200, method='.632', scoring_func=None, predict_proba=False, random_seed=None, clone_estimator=True, **fit_params)* Implementation of the .632 [1] and .632+ [2] bootstrap for supervised learning References: - [1] Efron, Bradley. 1983. \"Estimating the Error Rate of a Prediction Rule: Improvement on Cross-Validation.\" Journal of the American Statistical Association 78 (382): 316. doi:10.2307/2288636. - [2] Efron, Bradley, and Robert Tibshirani. 1997. \"Improvements on Cross-Validation: The .632+ Bootstrap Method.\" Journal of the American Statistical Association 92 (438): 548. doi:10.2307/2965703. **Parameters** - `estimator` : object An estimator for classification or regression that follows the scikit-learn API and implements \"fit\" and \"predict\" methods. - `X` : array-like The data to fit. Can be, for example a list, or an array at least 2d. - `y` : array-like, optional, default: None The target variable to try to predict in the case of supervised learning. - `n_splits` : int (default=200) Number of bootstrap iterations. Must be larger than 1. - `method` : str (default='.632') The bootstrap method, which can be either - 1) '.632' bootstrap (default) - 2) '.632+' bootstrap - 3) 'oob' (regular out-of-bag, no weighting) for comparison studies. - `scoring_func` : callable, Score function (or loss function) with signature ``scoring_func(y, y_pred, **kwargs)``. If none, uses classification accuracy if the estimator is a classifier and mean squared error if the estimator is a regressor. - `predict_proba` : bool Whether to use the `predict_proba` function for the `estimator` argument. This is to be used in conjunction with `scoring_func` which takes in probability values instead of actual predictions. For example, if the scoring_func is :meth:`sklearn.metrics.roc_auc_score`, then use `predict_proba=True`. Note that this requires `estimator` to have `predict_proba` method implemented. - `random_seed` : int (default=None) If int, random_seed is the seed used by the random number generator. - `clone_estimator` : bool (default=True) Clones the estimator if true, otherwise fits the original. - `fit_params` : additional parameters Additional parameters to be passed to the .fit() function of the estimator when it is fit to the bootstrap samples. **Returns** - `scores` : array of float, shape=(len(list(n_splits)),) Array of scores of the estimator for each bootstrap replicate. **Examples** >>> from sklearn import datasets, linear_model >>> from mlxtend.evaluate import bootstrap_point632_score >>> iris = datasets.load_iris() >>> X = iris.data >>> y = iris.target >>> lr = linear_model.LogisticRegression() >>> scores = bootstrap_point632_score(lr, X, y) >>> acc = np.mean(scores) >>> print('Accuracy:', acc) 0.953023146884 >>> lower = np.percentile(scores, 2.5) >>> upper = np.percentile(scores, 97.5) >>> print('95%% Confidence interval: [%.2f, %.2f]' % (lower, upper)) 95% Confidence interval: [0.90, 0.98] For more usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/bootstrap_point632_score/ ``` cochrans_q cochrans_q(y_target, y_model_predictions)* Cochran's Q test to compare 2 or more models. Parameters y_target : array-like, shape=[n_samples] True class labels as 1D NumPy array. *y_model_predictions : array-likes, shape=[n_samples] Variable number of 2 or more arrays that contain the predicted class labels from models as 1D NumPy array. Returns q, p : float or None, float Returns the Q (chi-squared) value and the p-value Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/cochrans_q/ combined_ftest_5x2cv combined_ftest_5x2cv(estimator1, estimator2, X, y, scoring=None, random_seed=None) Implements the 5x2cv combined F test proposed by Alpaydin 1999, to compare the performance of two models. Parameters estimator1 : scikit-learn classifier or regressor estimator2 : scikit-learn classifier or regressor X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values. scoring : str, callable, or None (default: None) If None (default), uses 'accuracy' for sklearn classifiers and 'r2' for sklearn regressors. If str, uses a sklearn scoring metric string identifier, for example {accuracy, f1, precision, recall, roc_auc} for classifiers, {'mean_absolute_error', 'mean_squared_error'/'neg_mean_squared_error', 'median_absolute_error', 'r2'} for regressors. If a callable object or function is provided, it has to be conform with sklearn's signature scorer(estimator, X, y) ; see https://scikit-learn.org/stable/modules/generated/sklearn.metrics.make_scorer.html for more information. random_seed : int or None (default: None) Random seed for creating the test/train splits. Returns f : float The F-statistic pvalue : float Two-tailed p-value. If the chosen significance level is larger than the p-value, we reject the null hypothesis and accept that there are significant differences in the two compared models. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/combined_ftest_5x2cv/ confusion_matrix confusion_matrix(y_target, y_predicted, binary=False, positive_label=1) Compute a confusion matrix/contingency table. Parameters y_target : array-like, shape=[n_samples] True class labels. y_predicted : array-like, shape=[n_samples] Predicted class labels. binary : bool (default: False) Maps a multi-class problem onto a binary confusion matrix, where the positive class is 1 and all other classes are 0. positive_label : int (default: 1) Class label of the positive class. Returns mat : array-like, shape=[n_classes, n_classes] Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/confusion_matrix/ create_counterfactual create_counterfactual(x_reference, y_desired, model, X_dataset, y_desired_proba=None, lammbda=0.1, random_seed=None) Implementation of the counterfactual method by Wachter et al. 2017 References: - Wachter, S., Mittelstadt, B., & Russell, C. (2017). Counterfactual explanations without opening the black box: Automated decisions and the GDPR. Harv. JL & Tech., 31, 841., https://arxiv.org/abs/1711.00399 Parameters x_reference : array-like, shape=[m_features] The data instance (training example) to be explained. y_desired : int The desired class label for x_reference . model : estimator A (scikit-learn) estimator implementing .predict() and/or predict_proba() . - If model supports predict_proba() , then this is used by default for the first loss term, (lambda * model.predict[_proba](x_counterfact) - y_desired[_proba])^2 - Otherwise, method will fall back to predict . X_dataset : array-like, shape=[n_examples, m_features] A (training) dataset for picking the initial counterfactual as initial value for starting the optimization procedure. y_desired_proba : float (default: None) A float within the range [0, 1] designating the desired class probability for y_desired . - If y_desired_proba=None (default), the first loss term is (lambda * model(x_counterfact) - y_desired)^2 where y_desired is a class label - If y_desired_proba is not None, the first loss term is (lambda * model(x_counterfact) - y_desired_proba)^2 lammbda : Weighting parameter for the first loss term, (lambda * model(x_counterfact) - y_desired[_proba])^2 random_seed : int (default=None) If int, random_seed is the seed used by the random number generator for selecting the inital counterfactual from X_dataset . feature_importance_permutation feature_importance_permutation(X, y, predict_method, metric, num_rounds=1, feature_groups=None, seed=None) Feature importance imputation via permutation importance Parameters X : NumPy array, shape = [n_samples, n_features] Dataset, where n_samples is the number of samples and n_features is the number of features. y : NumPy array, shape = [n_samples] Target values. predict_method : prediction function A callable function that predicts the target values from X. metric : str, callable The metric for evaluating the feature importance through permutation. By default, the strings 'accuracy' is recommended for classifiers and the string 'r2' is recommended for regressors. Optionally, a custom scoring function (e.g., metric=scoring_func ) that accepts two arguments, y_true and y_pred, which have similar shape to the y array. num_rounds : int (default=1) Number of rounds the feature columns are permuted to compute the permutation importance. feature_groups : list or None (default=None) Optional argument for treating certain features as a group. For example [1, 2, [3, 4, 5]] , which can be useful for interpretability, for example, if features 3, 4, 5 are one-hot encoded features. seed : int or None (default=None) Random seed for permuting the feature columns. Returns mean_importance_vals, all_importance_vals : NumPy arrays. The first array, mean_importance_vals has shape [n_features, ] and contains the importance values for all features. The shape of the second array is [n_features, num_rounds] and contains the feature importance for each repetition. If num_rounds=1, it contains the same values as the first array, mean_importance_vals. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/feature_importance_permutation/ ftest ftest(y_target, y_model_predictions)* F-Test test to compare 2 or more models. Parameters y_target : array-like, shape=[n_samples] True class labels as 1D NumPy array. *y_model_predictions : array-likes, shape=[n_samples] Variable number of 2 or more arrays that contain the predicted class labels from models as 1D NumPy array. Returns f, p : float or None, float Returns the F-value and the p-value Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/ftest/ lift_score lift_score(y_target, y_predicted, binary=True, positive_label=1) Lift measures the degree to which the predictions of a classification model are better than randomly-generated predictions. The in terms of True Positives (TP), True Negatives (TN), False Positives (FP), and False Negatives (FN), the lift score is computed as: [ TP / (TP+FP) ] / [ (TP+FN) / (TP+TN+FP+FN) ] Parameters y_target : array-like, shape=[n_samples] True class labels. y_predicted : array-like, shape=[n_samples] Predicted class labels. binary : bool (default: True) Maps a multi-class problem onto a binary, where the positive class is 1 and all other classes are 0. positive_label : int (default: 0) Class label of the positive class. Returns score : float Lift score in the range [0, infinity] Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/lift_score/ mcnemar mcnemar(ary, corrected=True, exact=False) McNemar test for paired nominal data Parameters ary : array-like, shape=[2, 2] 2 x 2 contigency table (as returned by evaluate.mcnemar_table), where a: ary[0, 0]: # of samples that both models predicted correctly b: ary[0, 1]: # of samples that model 1 got right and model 2 got wrong c: ary[1, 0]: # of samples that model 2 got right and model 1 got wrong d: aryCell [1, 1]: # of samples that both models predicted incorrectly corrected : array-like, shape=[n_samples] (default: True) Uses Edward's continuity correction for chi-squared if True exact : bool, (default: False) If True , uses an exact binomial test comparing b to a binomial distribution with n = b + c and p = 0.5. It is highly recommended to use exact=True for sample sizes < 25 since chi-squared is not well-approximated by the chi-squared distribution! Returns chi2, p : float or None, float Returns the chi-squared value and the p-value; if exact=True (default: False ), chi2 is None Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/mcnemar/ mcnemar_table mcnemar_table(y_target, y_model1, y_model2) Compute a 2x2 contigency table for McNemar's test. Parameters y_target : array-like, shape=[n_samples] True class labels as 1D NumPy array. y_model1 : array-like, shape=[n_samples] Predicted class labels from model as 1D NumPy array. y_model2 : array-like, shape=[n_samples] Predicted class labels from model 2 as 1D NumPy array. Returns tb : array-like, shape=[2, 2] 2x2 contingency table with the following contents: a: tb[0, 0]: # of samples that both models predicted correctly b: tb[0, 1]: # of samples that model 1 got right and model 2 got wrong c: tb[1, 0]: # of samples that model 2 got right and model 1 got wrong d: tb[1, 1]: # of samples that both models predicted incorrectly Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/mcnemar_table/ mcnemar_tables mcnemar_tables(y_target, y_model_predictions)* Compute multiple 2x2 contigency tables for McNemar's test or Cochran's Q test. Parameters y_target : array-like, shape=[n_samples] True class labels as 1D NumPy array. y_model_predictions : array-like, shape=[n_samples] Predicted class labels for a model. Returns tables : dict Dictionary of NumPy arrays with shape=[2, 2]. Each dictionary key names the two models to be compared based on the order the models were passed as *y_model_predictions . The number of dictionary entries is equal to the number of pairwise combinations between the m models, i.e., \"m choose 2.\" For example the following target array (containing the true labels) and 3 models y_true = np.array([0, 0, 0, 0, 0, 1, 1, 1, 1, 1]) y_mod0 = np.array([0, 1, 0, 0, 0, 1, 1, 0, 0, 0]) y_mod1 = np.array([0, 0, 1, 1, 0, 1, 1, 0, 0, 0]) y_mod2 = np.array([0, 1, 1, 1, 0, 1, 0, 0, 0, 0]) would result in the following dictionary: {'model_0 vs model_1': array([[ 4., 1.], [ 2., 3.]]), 'model_0 vs model_2': array([[ 3., 0.], [ 3., 4.]]), 'model_1 vs model_2': array([[ 3., 0.], [ 2., 5.]])} Each array is structured in the following way: tb[0, 0]: # of samples that both models predicted correctly tb[0, 1]: # of samples that model a got right and model b got wrong tb[1, 0]: # of samples that model b got right and model a got wrong tb[1, 1]: # of samples that both models predicted incorrectly Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/mcnemar_tables/ paired_ttest_5x2cv paired_ttest_5x2cv(estimator1, estimator2, X, y, scoring=None, random_seed=None) Implements the 5x2cv paired t test proposed by Dieterrich (1998) to compare the performance of two models. Parameters estimator1 : scikit-learn classifier or regressor estimator2 : scikit-learn classifier or regressor X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values. scoring : str, callable, or None (default: None) If None (default), uses 'accuracy' for sklearn classifiers and 'r2' for sklearn regressors. If str, uses a sklearn scoring metric string identifier, for example {accuracy, f1, precision, recall, roc_auc} for classifiers, {'mean_absolute_error', 'mean_squared_error'/'neg_mean_squared_error', 'median_absolute_error', 'r2'} for regressors. If a callable object or function is provided, it has to be conform with sklearn's signature scorer(estimator, X, y) ; see https://scikit-learn.org/stable/modules/generated/sklearn.metrics.make_scorer.html for more information. random_seed : int or None (default: None) Random seed for creating the test/train splits. Returns t : float The t-statistic pvalue : float Two-tailed p-value. If the chosen significance level is larger than the p-value, we reject the null hypothesis and accept that there are significant differences in the two compared models. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/paired_ttest_5x2cv/ paired_ttest_kfold_cv paired_ttest_kfold_cv(estimator1, estimator2, X, y, cv=10, scoring=None, shuffle=False, random_seed=None) Implements the k-fold paired t test procedure to compare the performance of two models. Parameters estimator1 : scikit-learn classifier or regressor estimator2 : scikit-learn classifier or regressor X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values. cv : int (default: 10) Number of splits and iteration for the cross-validation procedure scoring : str, callable, or None (default: None) If None (default), uses 'accuracy' for sklearn classifiers and 'r2' for sklearn regressors. If str, uses a sklearn scoring metric string identifier, for example {accuracy, f1, precision, recall, roc_auc} for classifiers, {'mean_absolute_error', 'mean_squared_error'/'neg_mean_squared_error', 'median_absolute_error', 'r2'} for regressors. If a callable object or function is provided, it has to be conform with sklearn's signature scorer(estimator, X, y) ; see https://scikit-learn.org/stable/modules/generated/sklearn.metrics.make_scorer.html for more information. shuffle : bool (default: True) Whether to shuffle the dataset for generating the k-fold splits. random_seed : int or None (default: None) Random seed for shuffling the dataset for generating the k-fold splits. Ignored if shuffle=False. Returns t : float The t-statistic pvalue : float Two-tailed p-value. If the chosen significance level is larger than the p-value, we reject the null hypothesis and accept that there are significant differences in the two compared models. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/paired_ttest_kfold_cv/ paired_ttest_resampled paired_ttest_resampled(estimator1, estimator2, X, y, num_rounds=30, test_size=0.3, scoring=None, random_seed=None) Implements the resampled paired t test procedure to compare the performance of two models (also called k-hold-out paired t test). Parameters estimator1 : scikit-learn classifier or regressor estimator2 : scikit-learn classifier or regressor X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values. num_rounds : int (default: 30) Number of resampling iterations (i.e., train/test splits) test_size : float or int (default: 0.3) If float, should be between 0.0 and 1.0 and represent the proportion of the dataset to use as a test set. If int, represents the absolute number of test exsamples. scoring : str, callable, or None (default: None) If None (default), uses 'accuracy' for sklearn classifiers and 'r2' for sklearn regressors. If str, uses a sklearn scoring metric string identifier, for example {accuracy, f1, precision, recall, roc_auc} for classifiers, {'mean_absolute_error', 'mean_squared_error'/'neg_mean_squared_error', 'median_absolute_error', 'r2'} for regressors. If a callable object or function is provided, it has to be conform with sklearn's signature scorer(estimator, X, y) ; see https://scikit-learn.org/stable/modules/generated/sklearn.metrics.make_scorer.html for more information. random_seed : int or None (default: None) Random seed for creating the test/train splits. Returns t : float The t-statistic pvalue : float Two-tailed p-value. If the chosen significance level is larger than the p-value, we reject the null hypothesis and accept that there are significant differences in the two compared models. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/paired_ttest_resampled/ permutation_test permutation_test(x, y, func='x_mean != y_mean', method='exact', num_rounds=1000, seed=None, paired=False) Nonparametric permutation test Parameters x : list or numpy array with shape (n_datapoints,) A list or 1D numpy array of the first sample (e.g., the treatment group). y : list or numpy array with shape (n_datapoints,) A list or 1D numpy array of the second sample (e.g., the control group). func : custom function or str (default: 'x_mean != y_mean') function to compute the statistic for the permutation test. - If 'x_mean != y_mean', uses func=lambda x, y: np.abs(np.mean(x) - np.mean(y))) for a two-sided test. - If 'x_mean > y_mean', uses func=lambda x, y: np.mean(x) - np.mean(y)) for a one-sided test. - If 'x_mean < y_mean', uses func=lambda x, y: np.mean(y) - np.mean(x)) for a one-sided test. method : 'approximate' or 'exact' (default: 'exact') If 'exact' (default), all possible permutations are considered. If 'approximate' the number of drawn samples is given by num_rounds . Note that 'exact' is typically not feasible unless the dataset size is relatively small. paired : bool If True, a paired test is performed by only exchanging each datapoint with its associate. num_rounds : int (default: 1000) The number of permutation samples if method='approximate' . seed : int or None (default: None) The random seed for generating permutation samples if method='approximate' . Returns p-value under the null hypothesis Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/permutation_test/ proportion_difference proportion_difference(proportion_1, proportion_2, n_1, n_2=None) Computes the test statistic and p-value for a difference of proportions test. Parameters proportion_1 : float The first proportion proportion_2 : float The second proportion n_1 : int The sample size of the first test sample n_2 : int or None (default=None) The sample size of the second test sample. If None , n_1 = n_2 . Returns z, p : float or None, float Returns the z-score and the p-value Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/proportion_difference/ scoring scoring(y_target, y_predicted, metric='error', positive_label=1, unique_labels='auto') Compute a scoring metric for supervised learning. Parameters y_target : array-like, shape=[n_values] True class labels or target values. y_predicted : array-like, shape=[n_values] Predicted class labels or target values. metric : str (default: 'error') Performance metric: 'accuracy': (TP + TN)/(FP + FN + TP + TN) = 1-ERR 'average per-class accuracy': Average per-class accuracy 'average per-class error': Average per-class error 'balanced per-class accuracy': Average per-class accuracy 'balanced per-class error': Average per-class error 'error': (TP + TN)/(FP+ FN + TP + TN) = 1-ACC 'false_positive_rate': FP/N = FP/(FP + TN) 'true_positive_rate': TP/P = TP/(FN + TP) 'true_negative_rate': TN/N = TN/(FP + TN) 'precision': TP/(TP + FP) 'recall': equal to 'true_positive_rate' 'sensitivity': equal to 'true_positive_rate' or 'recall' 'specificity': equal to 'true_negative_rate' 'f1': 2 * (PRE * REC)/(PRE + REC) 'matthews_corr_coef': (TP TN - FP FN) / (sqrt{(TP + FP)( TP + FN )( TN + FP )( TN + FN )}) Where: [TP: True positives, TN = True negatives, TN: True negatives, FN = False negatives] positive_label : int (default: 1) Label of the positive class for binary classification metrics. unique_labels : str or array-like (default: 'auto') If 'auto', deduces the unique class labels from y_target Returns score : float Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/scoring/","title":"Mlxtend.evaluate"},{"location":"api_subpackages/mlxtend.evaluate/#bootstrapoutofbag","text":"BootstrapOutOfBag(n_splits=200, random_seed=None) Parameters n_splits : int (default=200) Number of bootstrap iterations. Must be larger than 1. random_seed : int (default=None) If int, random_seed is the seed used by the random number generator. Returns train_idx : ndarray The training set indices for that split. test_idx : ndarray The testing set indices for that split. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/BootstrapOutOfBag/","title":"BootstrapOutOfBag"},{"location":"api_subpackages/mlxtend.evaluate/#methods","text":"get_n_splits(X=None, y=None, groups=None) Returns the number of splitting iterations in the cross-validator Parameters X : object Always ignored, exists for compatibility with scikit-learn. y : object Always ignored, exists for compatibility with scikit-learn. groups : object Always ignored, exists for compatibility with scikit-learn. Returns n_splits : int Returns the number of splitting iterations in the cross-validator. split(X, y=None, groups=None) y : array-like or None (default: None) Argument is not used and only included as parameter for compatibility, similar to KFold in scikit-learn. groups : array-like or None (default: None) Argument is not used and only included as parameter for compatibility, similar to KFold in scikit-learn.","title":"Methods"},{"location":"api_subpackages/mlxtend.evaluate/#grouptimeseriessplit","text":"GroupTimeSeriesSplit(test_size, train_size=None, n_splits=None, gap_size=0, shift_size=1, window_type='rolling') Group time series cross-validator. Parameters test_size : int Size of test dataset. train_size : int (default=None) Size of train dataset. n_splits : int (default=None) Number of the splits. gap_size : int (default=0) Gap size between train and test datasets. shift_size : int (default=1) Step to shift for the next fold. window_type : str (default=\"rolling\") Type of the window. Possible values: \"rolling\", \"expanding\". Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/GroupTimeSeriesSplit/","title":"GroupTimeSeriesSplit"},{"location":"api_subpackages/mlxtend.evaluate/#methods_1","text":"get_n_splits(X=None, y=None, groups=None) Returns the number of splitting iterations in the cross-validator. Parameters X : object Always ignored, exists for compatibility. y : object Always ignored, exists for compatibility. groups : object Always ignored, exists for compatibility. Returns n_splits : int Returns the number of splitting iterations in the cross-validator. split(X, y=None, groups=None) Generate indices to split data into training and test set. Parameters X : array-like Training data. y : array-like (default=None) Always ignored, exists for compatibility. groups : array-like (default=None) Array with group names or sequence numbers. Yields train : ndarray The training set indices for that split. test : ndarray The testing set indices for that split.","title":"Methods"},{"location":"api_subpackages/mlxtend.evaluate/#predefinedholdoutsplit","text":"PredefinedHoldoutSplit(valid_indices) Train/Validation set splitter for sklearn's GridSearchCV etc. Uses user-specified train/validation set indices to split a dataset into train/validation sets using user-defined or random indices. Parameters valid_indices : array-like, shape (num_examples,) Indices of the training examples in the training set to be used for validation. All other indices in the training set are used to for a training subset for model fitting. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/PredefinedHoldoutSplit/","title":"PredefinedHoldoutSplit"},{"location":"api_subpackages/mlxtend.evaluate/#methods_2","text":"get_n_splits(X=None, y=None, groups=None) Returns the number of splitting iterations in the cross-validator Parameters X : object Always ignored, exists for compatibility. y : object Always ignored, exists for compatibility. groups : object Always ignored, exists for compatibility. Returns n_splits : 1 Returns the number of splitting iterations in the cross-validator. Always returns 1. split(X, y, groups=None) Generate indices to split data into training and test set. Parameters X : array-like, shape (num_examples, num_features) Training data, where num_examples is the number of examples and num_features is the number of features. y : array-like, shape (num_examples,) The target variable for supervised learning problems. Stratification is done based on the y labels. groups : object Always ignored, exists for compatibility. Yields train_index : ndarray The training set indices for that split. valid_index : ndarray The validation set indices for that split.","title":"Methods"},{"location":"api_subpackages/mlxtend.evaluate/#randomholdoutsplit","text":"RandomHoldoutSplit(valid_size=0.5, random_seed=None, stratify=False) Train/Validation set splitter for sklearn's GridSearchCV etc. Provides train/validation set indices to split a dataset into train/validation sets using random indices. Parameters valid_size : float (default: 0.5) Proportion of examples that being assigned as validation examples. 1- valid_size will then automatically be assigned as training set examples. random_seed : int (default: None) The random seed for splitting the data into training and validation set partitions. stratify : bool (default: False) True or False, whether to perform a stratified split or not Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/RandomHoldoutSplit/","title":"RandomHoldoutSplit"},{"location":"api_subpackages/mlxtend.evaluate/#methods_3","text":"get_n_splits(X=None, y=None, groups=None) Returns the number of splitting iterations in the cross-validator Parameters X : object Always ignored, exists for compatibility. y : object Always ignored, exists for compatibility. groups : object Always ignored, exists for compatibility. Returns n_splits : 1 Returns the number of splitting iterations in the cross-validator. Always returns 1. split(X, y, groups=None) Generate indices to split data into training and test set. Parameters X : array-like, shape (num_examples, num_features) Training data, where num_examples is the number of training examples and num_features is the number of features. y : array-like, shape (num_examples,) The target variable for supervised learning problems. Stratification is done based on the y labels. groups : object Always ignored, exists for compatibility. Yields train_index : ndarray The training set indices for that split. valid_index : ndarray The validation set indices for that split.","title":"Methods"},{"location":"api_subpackages/mlxtend.evaluate/#accuracy_score","text":"accuracy_score(y_target, y_predicted, method='standard', pos_label=1, normalize=True) General accuracy function for supervised learning. Parameters y_target : array-like, shape=[n_values] True class labels or target values. y_predicted : array-like, shape=[n_values] Predicted class labels or target values. method : str, 'standard' by default. The chosen method for accuracy computation. If set to 'standard', computes overall accuracy. If set to 'binary', computes accuracy for class pos_label. If set to 'average', computes average per-class (balanced) accuracy. If set to 'balanced', computes the scikit-learn-style balanced accuracy. pos_label : str or int, 1 by default. The class whose accuracy score is to be reported. Used only when method is set to 'binary' normalize : bool, True by default. If True, returns fraction of correctly classified samples. If False, returns number of correctly classified samples. Returns score: float Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/accuracy_score/","title":"accuracy_score"},{"location":"api_subpackages/mlxtend.evaluate/#bias_variance_decomp","text":"bias_variance_decomp(estimator, X_train, y_train, X_test, y_test, loss='0-1_loss', num_rounds=200, random_seed=None, fit_params) estimator : object A classifier or regressor object or class implementing both a fit and predict method similar to the scikit-learn API. X_train : array-like, shape=(num_examples, num_features) A training dataset for drawing the bootstrap samples to carry out the bias-variance decomposition. y_train : array-like, shape=(num_examples) Targets (class labels, continuous values in case of regression) associated with the X_train examples. X_test : array-like, shape=(num_examples, num_features) The test dataset for computing the average loss, bias, and variance. y_test : array-like, shape=(num_examples) Targets (class labels, continuous values in case of regression) associated with the X_test examples. loss : str (default='0-1_loss') Loss function for performing the bias-variance decomposition. Currently allowed values are '0-1_loss' and 'mse'. num_rounds : int (default=200) Number of bootstrap rounds (sampling from the training set) for performing the bias-variance decomposition. Each bootstrap sample has the same size as the original training set. random_seed : int (default=None) Random seed for the bootstrap sampling used for the bias-variance decomposition. fit_params : additional parameters Additional parameters to be passed to the .fit() function of the estimator when it is fit to the bootstrap samples. Returns avg_expected_loss, avg_bias, avg_var : returns the average expected average bias, and average bias (all floats), where the average is computed over the data points in the test set. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/bias_variance_decomp/","title":"bias_variance_decomp"},{"location":"api_subpackages/mlxtend.evaluate/#bootstrap","text":"bootstrap(x, func, num_rounds=1000, ci=0.95, ddof=1, seed=None) Implements the ordinary nonparametric bootstrap Parameters x : NumPy array, shape=(n_samples, [n_columns]) An one or multidimensional array of data records func : A function which computes a statistic that is used to compute the bootstrap replicates (the statistic computed from the bootstrap samples). This function must return a scalar value. For example, np.mean or np.median would be an acceptable argument for func if x is a 1-dimensional array or vector. num_rounds : int (default=1000) The number of bootstrap samples to draw where each bootstrap sample has the same number of records as the original dataset. ci : int (default=0.95) An integer in the range (0, 1) that represents the confidence level for computing the confidence interval. For example, ci=0.95 (default) will compute the 95% confidence interval from the bootstrap replicates. ddof : int The delta degrees of freedom used when computing the standard error. seed : int or None (default=None) Random seed for generating bootstrap samples. Returns original, standard_error, (lower_ci, upper_ci) : tuple Returns the statistic of the original sample ( original ), the standard error of the estimate, and the respective confidence interval bounds. Examples ``` >>> from mlxtend.evaluate import bootstrap >>> rng = np.random.RandomState(123) >>> x = rng.normal(loc=5., size=100) >>> original, std_err, ci_bounds = bootstrap(x, ... num_rounds=1000, ... func=np.mean, ... ci=0.95, ... seed=123) >>> print('Mean: %.2f, SE: +/- %.2f, CI95: [%.2f, %.2f]' % (original, ... std_err, ... ci_bounds[0], ... ci_bounds[1])) Mean: 5.03, SE: +/- 0.11, CI95: [4.80, 5.26] >>> For more usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/bootstrap/ ## bootstrap_point632_score *bootstrap_point632_score(estimator, X, y, n_splits=200, method='.632', scoring_func=None, predict_proba=False, random_seed=None, clone_estimator=True, **fit_params)* Implementation of the .632 [1] and .632+ [2] bootstrap for supervised learning References: - [1] Efron, Bradley. 1983. \"Estimating the Error Rate of a Prediction Rule: Improvement on Cross-Validation.\" Journal of the American Statistical Association 78 (382): 316. doi:10.2307/2288636. - [2] Efron, Bradley, and Robert Tibshirani. 1997. \"Improvements on Cross-Validation: The .632+ Bootstrap Method.\" Journal of the American Statistical Association 92 (438): 548. doi:10.2307/2965703. **Parameters** - `estimator` : object An estimator for classification or regression that follows the scikit-learn API and implements \"fit\" and \"predict\" methods. - `X` : array-like The data to fit. Can be, for example a list, or an array at least 2d. - `y` : array-like, optional, default: None The target variable to try to predict in the case of supervised learning. - `n_splits` : int (default=200) Number of bootstrap iterations. Must be larger than 1. - `method` : str (default='.632') The bootstrap method, which can be either - 1) '.632' bootstrap (default) - 2) '.632+' bootstrap - 3) 'oob' (regular out-of-bag, no weighting) for comparison studies. - `scoring_func` : callable, Score function (or loss function) with signature ``scoring_func(y, y_pred, **kwargs)``. If none, uses classification accuracy if the estimator is a classifier and mean squared error if the estimator is a regressor. - `predict_proba` : bool Whether to use the `predict_proba` function for the `estimator` argument. This is to be used in conjunction with `scoring_func` which takes in probability values instead of actual predictions. For example, if the scoring_func is :meth:`sklearn.metrics.roc_auc_score`, then use `predict_proba=True`. Note that this requires `estimator` to have `predict_proba` method implemented. - `random_seed` : int (default=None) If int, random_seed is the seed used by the random number generator. - `clone_estimator` : bool (default=True) Clones the estimator if true, otherwise fits the original. - `fit_params` : additional parameters Additional parameters to be passed to the .fit() function of the estimator when it is fit to the bootstrap samples. **Returns** - `scores` : array of float, shape=(len(list(n_splits)),) Array of scores of the estimator for each bootstrap replicate. **Examples** >>> from sklearn import datasets, linear_model >>> from mlxtend.evaluate import bootstrap_point632_score >>> iris = datasets.load_iris() >>> X = iris.data >>> y = iris.target >>> lr = linear_model.LogisticRegression() >>> scores = bootstrap_point632_score(lr, X, y) >>> acc = np.mean(scores) >>> print('Accuracy:', acc) 0.953023146884 >>> lower = np.percentile(scores, 2.5) >>> upper = np.percentile(scores, 97.5) >>> print('95%% Confidence interval: [%.2f, %.2f]' % (lower, upper)) 95% Confidence interval: [0.90, 0.98] For more usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/bootstrap_point632_score/ ```","title":"bootstrap"},{"location":"api_subpackages/mlxtend.evaluate/#cochrans_q","text":"cochrans_q(y_target, y_model_predictions)* Cochran's Q test to compare 2 or more models. Parameters y_target : array-like, shape=[n_samples] True class labels as 1D NumPy array. *y_model_predictions : array-likes, shape=[n_samples] Variable number of 2 or more arrays that contain the predicted class labels from models as 1D NumPy array. Returns q, p : float or None, float Returns the Q (chi-squared) value and the p-value Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/cochrans_q/","title":"cochrans_q"},{"location":"api_subpackages/mlxtend.evaluate/#combined_ftest_5x2cv","text":"combined_ftest_5x2cv(estimator1, estimator2, X, y, scoring=None, random_seed=None) Implements the 5x2cv combined F test proposed by Alpaydin 1999, to compare the performance of two models. Parameters estimator1 : scikit-learn classifier or regressor estimator2 : scikit-learn classifier or regressor X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values. scoring : str, callable, or None (default: None) If None (default), uses 'accuracy' for sklearn classifiers and 'r2' for sklearn regressors. If str, uses a sklearn scoring metric string identifier, for example {accuracy, f1, precision, recall, roc_auc} for classifiers, {'mean_absolute_error', 'mean_squared_error'/'neg_mean_squared_error', 'median_absolute_error', 'r2'} for regressors. If a callable object or function is provided, it has to be conform with sklearn's signature scorer(estimator, X, y) ; see https://scikit-learn.org/stable/modules/generated/sklearn.metrics.make_scorer.html for more information. random_seed : int or None (default: None) Random seed for creating the test/train splits. Returns f : float The F-statistic pvalue : float Two-tailed p-value. If the chosen significance level is larger than the p-value, we reject the null hypothesis and accept that there are significant differences in the two compared models. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/combined_ftest_5x2cv/","title":"combined_ftest_5x2cv"},{"location":"api_subpackages/mlxtend.evaluate/#confusion_matrix","text":"confusion_matrix(y_target, y_predicted, binary=False, positive_label=1) Compute a confusion matrix/contingency table. Parameters y_target : array-like, shape=[n_samples] True class labels. y_predicted : array-like, shape=[n_samples] Predicted class labels. binary : bool (default: False) Maps a multi-class problem onto a binary confusion matrix, where the positive class is 1 and all other classes are 0. positive_label : int (default: 1) Class label of the positive class. Returns mat : array-like, shape=[n_classes, n_classes] Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/confusion_matrix/","title":"confusion_matrix"},{"location":"api_subpackages/mlxtend.evaluate/#create_counterfactual","text":"create_counterfactual(x_reference, y_desired, model, X_dataset, y_desired_proba=None, lammbda=0.1, random_seed=None) Implementation of the counterfactual method by Wachter et al. 2017 References: - Wachter, S., Mittelstadt, B., & Russell, C. (2017). Counterfactual explanations without opening the black box: Automated decisions and the GDPR. Harv. JL & Tech., 31, 841., https://arxiv.org/abs/1711.00399 Parameters x_reference : array-like, shape=[m_features] The data instance (training example) to be explained. y_desired : int The desired class label for x_reference . model : estimator A (scikit-learn) estimator implementing .predict() and/or predict_proba() . - If model supports predict_proba() , then this is used by default for the first loss term, (lambda * model.predict[_proba](x_counterfact) - y_desired[_proba])^2 - Otherwise, method will fall back to predict . X_dataset : array-like, shape=[n_examples, m_features] A (training) dataset for picking the initial counterfactual as initial value for starting the optimization procedure. y_desired_proba : float (default: None) A float within the range [0, 1] designating the desired class probability for y_desired . - If y_desired_proba=None (default), the first loss term is (lambda * model(x_counterfact) - y_desired)^2 where y_desired is a class label - If y_desired_proba is not None, the first loss term is (lambda * model(x_counterfact) - y_desired_proba)^2 lammbda : Weighting parameter for the first loss term, (lambda * model(x_counterfact) - y_desired[_proba])^2 random_seed : int (default=None) If int, random_seed is the seed used by the random number generator for selecting the inital counterfactual from X_dataset .","title":"create_counterfactual"},{"location":"api_subpackages/mlxtend.evaluate/#feature_importance_permutation","text":"feature_importance_permutation(X, y, predict_method, metric, num_rounds=1, feature_groups=None, seed=None) Feature importance imputation via permutation importance Parameters X : NumPy array, shape = [n_samples, n_features] Dataset, where n_samples is the number of samples and n_features is the number of features. y : NumPy array, shape = [n_samples] Target values. predict_method : prediction function A callable function that predicts the target values from X. metric : str, callable The metric for evaluating the feature importance through permutation. By default, the strings 'accuracy' is recommended for classifiers and the string 'r2' is recommended for regressors. Optionally, a custom scoring function (e.g., metric=scoring_func ) that accepts two arguments, y_true and y_pred, which have similar shape to the y array. num_rounds : int (default=1) Number of rounds the feature columns are permuted to compute the permutation importance. feature_groups : list or None (default=None) Optional argument for treating certain features as a group. For example [1, 2, [3, 4, 5]] , which can be useful for interpretability, for example, if features 3, 4, 5 are one-hot encoded features. seed : int or None (default=None) Random seed for permuting the feature columns. Returns mean_importance_vals, all_importance_vals : NumPy arrays. The first array, mean_importance_vals has shape [n_features, ] and contains the importance values for all features. The shape of the second array is [n_features, num_rounds] and contains the feature importance for each repetition. If num_rounds=1, it contains the same values as the first array, mean_importance_vals. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/feature_importance_permutation/","title":"feature_importance_permutation"},{"location":"api_subpackages/mlxtend.evaluate/#ftest","text":"ftest(y_target, y_model_predictions)* F-Test test to compare 2 or more models. Parameters y_target : array-like, shape=[n_samples] True class labels as 1D NumPy array. *y_model_predictions : array-likes, shape=[n_samples] Variable number of 2 or more arrays that contain the predicted class labels from models as 1D NumPy array. Returns f, p : float or None, float Returns the F-value and the p-value Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/ftest/","title":"ftest"},{"location":"api_subpackages/mlxtend.evaluate/#lift_score","text":"lift_score(y_target, y_predicted, binary=True, positive_label=1) Lift measures the degree to which the predictions of a classification model are better than randomly-generated predictions. The in terms of True Positives (TP), True Negatives (TN), False Positives (FP), and False Negatives (FN), the lift score is computed as: [ TP / (TP+FP) ] / [ (TP+FN) / (TP+TN+FP+FN) ] Parameters y_target : array-like, shape=[n_samples] True class labels. y_predicted : array-like, shape=[n_samples] Predicted class labels. binary : bool (default: True) Maps a multi-class problem onto a binary, where the positive class is 1 and all other classes are 0. positive_label : int (default: 0) Class label of the positive class. Returns score : float Lift score in the range [0, infinity] Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/lift_score/","title":"lift_score"},{"location":"api_subpackages/mlxtend.evaluate/#mcnemar","text":"mcnemar(ary, corrected=True, exact=False) McNemar test for paired nominal data Parameters ary : array-like, shape=[2, 2] 2 x 2 contigency table (as returned by evaluate.mcnemar_table), where a: ary[0, 0]: # of samples that both models predicted correctly b: ary[0, 1]: # of samples that model 1 got right and model 2 got wrong c: ary[1, 0]: # of samples that model 2 got right and model 1 got wrong d: aryCell [1, 1]: # of samples that both models predicted incorrectly corrected : array-like, shape=[n_samples] (default: True) Uses Edward's continuity correction for chi-squared if True exact : bool, (default: False) If True , uses an exact binomial test comparing b to a binomial distribution with n = b + c and p = 0.5. It is highly recommended to use exact=True for sample sizes < 25 since chi-squared is not well-approximated by the chi-squared distribution! Returns chi2, p : float or None, float Returns the chi-squared value and the p-value; if exact=True (default: False ), chi2 is None Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/mcnemar/","title":"mcnemar"},{"location":"api_subpackages/mlxtend.evaluate/#mcnemar_table","text":"mcnemar_table(y_target, y_model1, y_model2) Compute a 2x2 contigency table for McNemar's test. Parameters y_target : array-like, shape=[n_samples] True class labels as 1D NumPy array. y_model1 : array-like, shape=[n_samples] Predicted class labels from model as 1D NumPy array. y_model2 : array-like, shape=[n_samples] Predicted class labels from model 2 as 1D NumPy array. Returns tb : array-like, shape=[2, 2] 2x2 contingency table with the following contents: a: tb[0, 0]: # of samples that both models predicted correctly b: tb[0, 1]: # of samples that model 1 got right and model 2 got wrong c: tb[1, 0]: # of samples that model 2 got right and model 1 got wrong d: tb[1, 1]: # of samples that both models predicted incorrectly Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/mcnemar_table/","title":"mcnemar_table"},{"location":"api_subpackages/mlxtend.evaluate/#mcnemar_tables","text":"mcnemar_tables(y_target, y_model_predictions)* Compute multiple 2x2 contigency tables for McNemar's test or Cochran's Q test. Parameters y_target : array-like, shape=[n_samples] True class labels as 1D NumPy array. y_model_predictions : array-like, shape=[n_samples] Predicted class labels for a model. Returns tables : dict Dictionary of NumPy arrays with shape=[2, 2]. Each dictionary key names the two models to be compared based on the order the models were passed as *y_model_predictions . The number of dictionary entries is equal to the number of pairwise combinations between the m models, i.e., \"m choose 2.\" For example the following target array (containing the true labels) and 3 models y_true = np.array([0, 0, 0, 0, 0, 1, 1, 1, 1, 1]) y_mod0 = np.array([0, 1, 0, 0, 0, 1, 1, 0, 0, 0]) y_mod1 = np.array([0, 0, 1, 1, 0, 1, 1, 0, 0, 0]) y_mod2 = np.array([0, 1, 1, 1, 0, 1, 0, 0, 0, 0]) would result in the following dictionary: {'model_0 vs model_1': array([[ 4., 1.], [ 2., 3.]]), 'model_0 vs model_2': array([[ 3., 0.], [ 3., 4.]]), 'model_1 vs model_2': array([[ 3., 0.], [ 2., 5.]])} Each array is structured in the following way: tb[0, 0]: # of samples that both models predicted correctly tb[0, 1]: # of samples that model a got right and model b got wrong tb[1, 0]: # of samples that model b got right and model a got wrong tb[1, 1]: # of samples that both models predicted incorrectly Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/mcnemar_tables/","title":"mcnemar_tables"},{"location":"api_subpackages/mlxtend.evaluate/#paired_ttest_5x2cv","text":"paired_ttest_5x2cv(estimator1, estimator2, X, y, scoring=None, random_seed=None) Implements the 5x2cv paired t test proposed by Dieterrich (1998) to compare the performance of two models. Parameters estimator1 : scikit-learn classifier or regressor estimator2 : scikit-learn classifier or regressor X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values. scoring : str, callable, or None (default: None) If None (default), uses 'accuracy' for sklearn classifiers and 'r2' for sklearn regressors. If str, uses a sklearn scoring metric string identifier, for example {accuracy, f1, precision, recall, roc_auc} for classifiers, {'mean_absolute_error', 'mean_squared_error'/'neg_mean_squared_error', 'median_absolute_error', 'r2'} for regressors. If a callable object or function is provided, it has to be conform with sklearn's signature scorer(estimator, X, y) ; see https://scikit-learn.org/stable/modules/generated/sklearn.metrics.make_scorer.html for more information. random_seed : int or None (default: None) Random seed for creating the test/train splits. Returns t : float The t-statistic pvalue : float Two-tailed p-value. If the chosen significance level is larger than the p-value, we reject the null hypothesis and accept that there are significant differences in the two compared models. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/paired_ttest_5x2cv/","title":"paired_ttest_5x2cv"},{"location":"api_subpackages/mlxtend.evaluate/#paired_ttest_kfold_cv","text":"paired_ttest_kfold_cv(estimator1, estimator2, X, y, cv=10, scoring=None, shuffle=False, random_seed=None) Implements the k-fold paired t test procedure to compare the performance of two models. Parameters estimator1 : scikit-learn classifier or regressor estimator2 : scikit-learn classifier or regressor X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values. cv : int (default: 10) Number of splits and iteration for the cross-validation procedure scoring : str, callable, or None (default: None) If None (default), uses 'accuracy' for sklearn classifiers and 'r2' for sklearn regressors. If str, uses a sklearn scoring metric string identifier, for example {accuracy, f1, precision, recall, roc_auc} for classifiers, {'mean_absolute_error', 'mean_squared_error'/'neg_mean_squared_error', 'median_absolute_error', 'r2'} for regressors. If a callable object or function is provided, it has to be conform with sklearn's signature scorer(estimator, X, y) ; see https://scikit-learn.org/stable/modules/generated/sklearn.metrics.make_scorer.html for more information. shuffle : bool (default: True) Whether to shuffle the dataset for generating the k-fold splits. random_seed : int or None (default: None) Random seed for shuffling the dataset for generating the k-fold splits. Ignored if shuffle=False. Returns t : float The t-statistic pvalue : float Two-tailed p-value. If the chosen significance level is larger than the p-value, we reject the null hypothesis and accept that there are significant differences in the two compared models. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/paired_ttest_kfold_cv/","title":"paired_ttest_kfold_cv"},{"location":"api_subpackages/mlxtend.evaluate/#paired_ttest_resampled","text":"paired_ttest_resampled(estimator1, estimator2, X, y, num_rounds=30, test_size=0.3, scoring=None, random_seed=None) Implements the resampled paired t test procedure to compare the performance of two models (also called k-hold-out paired t test). Parameters estimator1 : scikit-learn classifier or regressor estimator2 : scikit-learn classifier or regressor X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values. num_rounds : int (default: 30) Number of resampling iterations (i.e., train/test splits) test_size : float or int (default: 0.3) If float, should be between 0.0 and 1.0 and represent the proportion of the dataset to use as a test set. If int, represents the absolute number of test exsamples. scoring : str, callable, or None (default: None) If None (default), uses 'accuracy' for sklearn classifiers and 'r2' for sklearn regressors. If str, uses a sklearn scoring metric string identifier, for example {accuracy, f1, precision, recall, roc_auc} for classifiers, {'mean_absolute_error', 'mean_squared_error'/'neg_mean_squared_error', 'median_absolute_error', 'r2'} for regressors. If a callable object or function is provided, it has to be conform with sklearn's signature scorer(estimator, X, y) ; see https://scikit-learn.org/stable/modules/generated/sklearn.metrics.make_scorer.html for more information. random_seed : int or None (default: None) Random seed for creating the test/train splits. Returns t : float The t-statistic pvalue : float Two-tailed p-value. If the chosen significance level is larger than the p-value, we reject the null hypothesis and accept that there are significant differences in the two compared models. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/paired_ttest_resampled/","title":"paired_ttest_resampled"},{"location":"api_subpackages/mlxtend.evaluate/#permutation_test","text":"permutation_test(x, y, func='x_mean != y_mean', method='exact', num_rounds=1000, seed=None, paired=False) Nonparametric permutation test Parameters x : list or numpy array with shape (n_datapoints,) A list or 1D numpy array of the first sample (e.g., the treatment group). y : list or numpy array with shape (n_datapoints,) A list or 1D numpy array of the second sample (e.g., the control group). func : custom function or str (default: 'x_mean != y_mean') function to compute the statistic for the permutation test. - If 'x_mean != y_mean', uses func=lambda x, y: np.abs(np.mean(x) - np.mean(y))) for a two-sided test. - If 'x_mean > y_mean', uses func=lambda x, y: np.mean(x) - np.mean(y)) for a one-sided test. - If 'x_mean < y_mean', uses func=lambda x, y: np.mean(y) - np.mean(x)) for a one-sided test. method : 'approximate' or 'exact' (default: 'exact') If 'exact' (default), all possible permutations are considered. If 'approximate' the number of drawn samples is given by num_rounds . Note that 'exact' is typically not feasible unless the dataset size is relatively small. paired : bool If True, a paired test is performed by only exchanging each datapoint with its associate. num_rounds : int (default: 1000) The number of permutation samples if method='approximate' . seed : int or None (default: None) The random seed for generating permutation samples if method='approximate' . Returns p-value under the null hypothesis Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/permutation_test/","title":"permutation_test"},{"location":"api_subpackages/mlxtend.evaluate/#proportion_difference","text":"proportion_difference(proportion_1, proportion_2, n_1, n_2=None) Computes the test statistic and p-value for a difference of proportions test. Parameters proportion_1 : float The first proportion proportion_2 : float The second proportion n_1 : int The sample size of the first test sample n_2 : int or None (default=None) The sample size of the second test sample. If None , n_1 = n_2 . Returns z, p : float or None, float Returns the z-score and the p-value Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/proportion_difference/","title":"proportion_difference"},{"location":"api_subpackages/mlxtend.evaluate/#scoring","text":"scoring(y_target, y_predicted, metric='error', positive_label=1, unique_labels='auto') Compute a scoring metric for supervised learning. Parameters y_target : array-like, shape=[n_values] True class labels or target values. y_predicted : array-like, shape=[n_values] Predicted class labels or target values. metric : str (default: 'error') Performance metric: 'accuracy': (TP + TN)/(FP + FN + TP + TN) = 1-ERR 'average per-class accuracy': Average per-class accuracy 'average per-class error': Average per-class error 'balanced per-class accuracy': Average per-class accuracy 'balanced per-class error': Average per-class error 'error': (TP + TN)/(FP+ FN + TP + TN) = 1-ACC 'false_positive_rate': FP/N = FP/(FP + TN) 'true_positive_rate': TP/P = TP/(FN + TP) 'true_negative_rate': TN/N = TN/(FP + TN) 'precision': TP/(TP + FP) 'recall': equal to 'true_positive_rate' 'sensitivity': equal to 'true_positive_rate' or 'recall' 'specificity': equal to 'true_negative_rate' 'f1': 2 * (PRE * REC)/(PRE + REC) 'matthews_corr_coef': (TP TN - FP FN) / (sqrt{(TP + FP)( TP + FN )( TN + FP )( TN + FN )}) Where: [TP: True positives, TN = True negatives, TN: True negatives, FN = False negatives] positive_label : int (default: 1) Label of the positive class for binary classification metrics. unique_labels : str or array-like (default: 'auto') If 'auto', deduces the unique class labels from y_target Returns score : float Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/scoring/","title":"scoring"},{"location":"api_subpackages/mlxtend.externals/","text":"mlxtend version: 0.23.1","title":"Mlxtend.externals"},{"location":"api_subpackages/mlxtend.feature_extraction/","text":"mlxtend version: 0.23.1 LinearDiscriminantAnalysis LinearDiscriminantAnalysis(n_discriminants=None) Linear Discriminant Analysis Class Parameters n_discriminants : int (default: None) The number of discrimants for transformation. Keeps the original dimensions of the dataset if None . Attributes w_ : array-like, shape=[n_features, n_discriminants] Projection matrix e_vals_ : array-like, shape=[n_features] Eigenvalues in sorted order. e_vecs_ : array-like, shape=[n_features] Eigenvectors in sorted order. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/feature_extraction/LinearDiscriminantAnalysis/ Methods fit(X, y, n_classes=None) Fit the LDA model with X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values. n_classes : int (default: None) A positive integer to declare the number of class labels if not all class labels are present in a partial training set. Gets the number of class labels automatically if None. Returns self : object get_params(deep=True) Get parameters for this estimator. Parameters deep : boolean, optional If True, will return the parameters for this estimator and contained subobjects that are estimators. Returns params : mapping of string to any Parameter names mapped to their values.' adapted from https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/base.py Author: Gael Varoquaux gael.varoquaux@normalesup.org License: BSD 3 clause set_params( params) Set the parameters of this estimator. The method works on simple estimators as well as on nested objects (such as pipelines). The latter have parameters of the form <component>__<parameter> so that it's possible to update each component of a nested object. Returns self adapted from https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/base.py Author: Gael Varoquaux <gael.varoquaux@normalesup.org> License: BSD 3 clause transform(X) Apply the linear transformation on X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns X_projected : np.ndarray, shape = [n_samples, n_discriminants] Projected training vectors. PrincipalComponentAnalysis PrincipalComponentAnalysis(n_components=None, solver='svd', whitening=False) Principal Component Analysis Class Parameters n_components : int (default: None) The number of principal components for transformation. Keeps the original dimensions of the dataset if None . solver : str (default: 'svd') Method for performing the matrix decomposition. {'eigen', 'svd'} whitening : bool (default: False) Performs whitening such that the covariance matrix of the transformed data will be the identity matrix. Attributes w_ : array-like, shape=[n_features, n_components] Projection matrix e_vals_ : array-like, shape=[n_features] Eigenvalues in sorted order. e_vecs_ : array-like, shape=[n_features] Eigenvectors in sorted order. e_vals_normalized_ : array-like, shape=[n_features] Normalized eigen values such that they sum up to 1. This is equal to what's often referred to as \"explained variance ratios.\" loadings_ : array_like, shape=[n_features, n_features] The factor loadings of the original variables onto the principal components. The columns are the principal components, and the rows are the features loadings. For instance, the first column contains the loadings onto the first principal component. Note that the signs may be flipped depending on whether you use the 'eigen' or 'svd' solver; this does not affect the interpretation of the loadings though. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/feature_extraction/PrincipalComponentAnalysis/ Methods fit(X, y=None) Learn model from training data. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns self : object get_params(deep=True) Get parameters for this estimator. Parameters deep : boolean, optional If True, will return the parameters for this estimator and contained subobjects that are estimators. Returns params : mapping of string to any Parameter names mapped to their values.' adapted from https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/base.py Author: Gael Varoquaux gael.varoquaux@normalesup.org License: BSD 3 clause set_params( params) Set the parameters of this estimator. The method works on simple estimators as well as on nested objects (such as pipelines). The latter have parameters of the form <component>__<parameter> so that it's possible to update each component of a nested object. Returns self adapted from https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/base.py Author: Gael Varoquaux <gael.varoquaux@normalesup.org> License: BSD 3 clause transform(X) Apply the linear transformation on X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns X_projected : np.ndarray, shape = [n_samples, n_components] Projected training vectors. RBFKernelPCA RBFKernelPCA(gamma=15.0, n_components=None, copy_X=True) RBF Kernel Principal Component Analysis for dimensionality reduction. Parameters gamma : float (default: 15.0) Free parameter (coefficient) of the RBF kernel. n_components : int (default: None) The number of principal components for transformation. Keeps the original dimensions of the dataset if None . copy_X : bool (default: True) Copies training data, which is required to compute the projection of new data via the transform method. Uses a reference to X if False. Attributes e_vals_ : array-like, shape=[n_features] Eigenvalues in sorted order. e_vecs_ : array-like, shape=[n_features] Eigenvectors in sorted order. X_projected_ : array-like, shape=[n_samples, n_components] Training samples projected along the component axes. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/feature_extraction/RBFKernelPCA/ Methods fit(X) Learn model from training data. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns self : object get_params(deep=True) Get parameters for this estimator. Parameters deep : boolean, optional If True, will return the parameters for this estimator and contained subobjects that are estimators. Returns params : mapping of string to any Parameter names mapped to their values.' adapted from https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/base.py Author: Gael Varoquaux gael.varoquaux@normalesup.org License: BSD 3 clause set_params( params) Set the parameters of this estimator. The method works on simple estimators as well as on nested objects (such as pipelines). The latter have parameters of the form <component>__<parameter> so that it's possible to update each component of a nested object. Returns self adapted from https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/base.py Author: Gael Varoquaux <gael.varoquaux@normalesup.org> License: BSD 3 clause transform(X) Apply the non-linear transformation on X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns X_projected : np.ndarray, shape = [n_samples, n_components] Projected training vectors.","title":"Mlxtend.feature extraction"},{"location":"api_subpackages/mlxtend.feature_extraction/#lineardiscriminantanalysis","text":"LinearDiscriminantAnalysis(n_discriminants=None) Linear Discriminant Analysis Class Parameters n_discriminants : int (default: None) The number of discrimants for transformation. Keeps the original dimensions of the dataset if None . Attributes w_ : array-like, shape=[n_features, n_discriminants] Projection matrix e_vals_ : array-like, shape=[n_features] Eigenvalues in sorted order. e_vecs_ : array-like, shape=[n_features] Eigenvectors in sorted order. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/feature_extraction/LinearDiscriminantAnalysis/","title":"LinearDiscriminantAnalysis"},{"location":"api_subpackages/mlxtend.feature_extraction/#methods","text":"fit(X, y, n_classes=None) Fit the LDA model with X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values. n_classes : int (default: None) A positive integer to declare the number of class labels if not all class labels are present in a partial training set. Gets the number of class labels automatically if None. Returns self : object get_params(deep=True) Get parameters for this estimator. Parameters deep : boolean, optional If True, will return the parameters for this estimator and contained subobjects that are estimators. Returns params : mapping of string to any Parameter names mapped to their values.' adapted from https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/base.py Author: Gael Varoquaux gael.varoquaux@normalesup.org License: BSD 3 clause set_params( params) Set the parameters of this estimator. The method works on simple estimators as well as on nested objects (such as pipelines). The latter have parameters of the form <component>__<parameter> so that it's possible to update each component of a nested object. Returns self adapted from https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/base.py Author: Gael Varoquaux <gael.varoquaux@normalesup.org> License: BSD 3 clause transform(X) Apply the linear transformation on X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns X_projected : np.ndarray, shape = [n_samples, n_discriminants] Projected training vectors.","title":"Methods"},{"location":"api_subpackages/mlxtend.feature_extraction/#principalcomponentanalysis","text":"PrincipalComponentAnalysis(n_components=None, solver='svd', whitening=False) Principal Component Analysis Class Parameters n_components : int (default: None) The number of principal components for transformation. Keeps the original dimensions of the dataset if None . solver : str (default: 'svd') Method for performing the matrix decomposition. {'eigen', 'svd'} whitening : bool (default: False) Performs whitening such that the covariance matrix of the transformed data will be the identity matrix. Attributes w_ : array-like, shape=[n_features, n_components] Projection matrix e_vals_ : array-like, shape=[n_features] Eigenvalues in sorted order. e_vecs_ : array-like, shape=[n_features] Eigenvectors in sorted order. e_vals_normalized_ : array-like, shape=[n_features] Normalized eigen values such that they sum up to 1. This is equal to what's often referred to as \"explained variance ratios.\" loadings_ : array_like, shape=[n_features, n_features] The factor loadings of the original variables onto the principal components. The columns are the principal components, and the rows are the features loadings. For instance, the first column contains the loadings onto the first principal component. Note that the signs may be flipped depending on whether you use the 'eigen' or 'svd' solver; this does not affect the interpretation of the loadings though. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/feature_extraction/PrincipalComponentAnalysis/","title":"PrincipalComponentAnalysis"},{"location":"api_subpackages/mlxtend.feature_extraction/#methods_1","text":"fit(X, y=None) Learn model from training data. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns self : object get_params(deep=True) Get parameters for this estimator. Parameters deep : boolean, optional If True, will return the parameters for this estimator and contained subobjects that are estimators. Returns params : mapping of string to any Parameter names mapped to their values.' adapted from https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/base.py Author: Gael Varoquaux gael.varoquaux@normalesup.org License: BSD 3 clause set_params( params) Set the parameters of this estimator. The method works on simple estimators as well as on nested objects (such as pipelines). The latter have parameters of the form <component>__<parameter> so that it's possible to update each component of a nested object. Returns self adapted from https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/base.py Author: Gael Varoquaux <gael.varoquaux@normalesup.org> License: BSD 3 clause transform(X) Apply the linear transformation on X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns X_projected : np.ndarray, shape = [n_samples, n_components] Projected training vectors.","title":"Methods"},{"location":"api_subpackages/mlxtend.feature_extraction/#rbfkernelpca","text":"RBFKernelPCA(gamma=15.0, n_components=None, copy_X=True) RBF Kernel Principal Component Analysis for dimensionality reduction. Parameters gamma : float (default: 15.0) Free parameter (coefficient) of the RBF kernel. n_components : int (default: None) The number of principal components for transformation. Keeps the original dimensions of the dataset if None . copy_X : bool (default: True) Copies training data, which is required to compute the projection of new data via the transform method. Uses a reference to X if False. Attributes e_vals_ : array-like, shape=[n_features] Eigenvalues in sorted order. e_vecs_ : array-like, shape=[n_features] Eigenvectors in sorted order. X_projected_ : array-like, shape=[n_samples, n_components] Training samples projected along the component axes. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/feature_extraction/RBFKernelPCA/","title":"RBFKernelPCA"},{"location":"api_subpackages/mlxtend.feature_extraction/#methods_2","text":"fit(X) Learn model from training data. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns self : object get_params(deep=True) Get parameters for this estimator. Parameters deep : boolean, optional If True, will return the parameters for this estimator and contained subobjects that are estimators. Returns params : mapping of string to any Parameter names mapped to their values.' adapted from https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/base.py Author: Gael Varoquaux gael.varoquaux@normalesup.org License: BSD 3 clause set_params( params) Set the parameters of this estimator. The method works on simple estimators as well as on nested objects (such as pipelines). The latter have parameters of the form <component>__<parameter> so that it's possible to update each component of a nested object. Returns self adapted from https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/base.py Author: Gael Varoquaux <gael.varoquaux@normalesup.org> License: BSD 3 clause transform(X) Apply the non-linear transformation on X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns X_projected : np.ndarray, shape = [n_samples, n_components] Projected training vectors.","title":"Methods"},{"location":"api_subpackages/mlxtend.feature_selection/","text":"mlxtend version: 0.23.1 ColumnSelector ColumnSelector(cols=None, drop_axis=False) Object for selecting specific columns from a data set. Parameters cols : array-like (default: None) A list specifying the feature indices to be selected. For example, [1, 4, 5] to select the 2nd, 5th, and 6th feature columns, and ['A','C','D'] to select the name of feature columns A, C and D. If None, returns all columns in the array. drop_axis : bool (default=False) Drops last axis if True and the only one column is selected. This is useful, e.g., when the ColumnSelector is used for selecting only one column and the resulting array should be fed to e.g., a scikit-learn column selector. E.g., instead of returning an array with shape (n_samples, 1), drop_axis=True will return an aray with shape (n_samples,). Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/feature_selection/ColumnSelector/ Methods fit(X, y=None) Mock method. Does nothing. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] (default: None) Returns self fit_transform(X, y=None) Return a slice of the input array. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] (default: None) Returns X_slice : shape = [n_samples, k_features] Subset of the feature space where k_features <= n_features get_params(deep=True) Get parameters for this estimator. Parameters deep : bool, default=True If True, will return the parameters for this estimator and contained subobjects that are estimators. Returns params : dict Parameter names mapped to their values. set_params( params) Set the parameters of this estimator. The method works on simple estimators as well as on nested objects (such as :class:`~sklearn.pipeline.Pipeline`). The latter have parameters of the form ``<component>__<parameter>`` so that it's possible to update each component of a nested object. Parameters **params : dict Estimator parameters. Returns self : estimator instance Estimator instance. transform(X, y=None) Return a slice of the input array. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] (default: None) Returns X_slice : shape = [n_samples, k_features] Subset of the feature space where k_features <= n_features ExhaustiveFeatureSelector ExhaustiveFeatureSelector(estimator, min_features=1, max_features=1, print_progress=True, scoring='accuracy', cv=5, n_jobs=1, pre_dispatch='2 n_jobs', clone_estimator=True, fixed_features=None, feature_groups=None)* Exhaustive Feature Selection for Classification and Regression. (new in v0.4.3) Parameters estimator : scikit-learn classifier or regressor min_features : int (default: 1) Minumum number of features to select max_features : int (default: 1) Maximum number of features to select. If parameter feature_groups is not None, the number of features is equal to the number of feature groups, i.e. len(feature_groups) . For example, if feature_groups = [[0], [1], [2, 3], [4]] , then the max_features value cannot exceed 4. print_progress : bool (default: True) Prints progress as the number of epochs to stderr. scoring : str, (default='accuracy') Scoring metric in {accuracy, f1, precision, recall, roc_auc} for classifiers, {'mean_absolute_error', 'mean_squared_error', 'median_absolute_error', 'r2'} for regressors, or a callable object or function with signature scorer(estimator, X, y) . cv : int (default: 5) Scikit-learn cross-validation generator or int . If estimator is a classifier (or y consists of integer class labels), stratified k-fold is performed, and regular k-fold cross-validation otherwise. No cross-validation if cv is None, False, or 0. n_jobs : int (default: 1) The number of CPUs to use for evaluating different feature subsets in parallel. -1 means 'all CPUs'. pre_dispatch : int, or string (default: '2*n_jobs') Controls the number of jobs that get dispatched during parallel execution if n_jobs > 1 or n_jobs=-1 . Reducing this number can be useful to avoid an explosion of memory consumption when more jobs get dispatched than CPUs can process. This parameter can be: None, in which case all the jobs are immediately created and spawned. Use this for lightweight and fast-running jobs, to avoid delays due to on-demand spawning of the jobs An int, giving the exact number of total jobs that are spawned A string, giving an expression as a function of n_jobs, as in 2*n_jobs clone_estimator : bool (default: True) Clones estimator if True; works with the original estimator instance if False. Set to False if the estimator doesn't implement scikit-learn's set_params and get_params methods. In addition, it is required to set cv=0, and n_jobs=1. fixed_features : tuple (default: None) If not None , the feature indices provided as a tuple will be regarded as fixed by the feature selector. For example, if fixed_features=(1, 3, 7) , the 2nd, 4th, and 8th feature are guaranteed to be present in the solution. Note that if fixed_features is not None , make sure that the number of features to be selected is greater than len(fixed_features) . In other words, ensure that k_features > len(fixed_features) . feature_groups : list or None (default: None) Optional argument for treating certain features as a group. This means, the features within a group are always selected together, never split. For example, feature_groups=[[1], [2], [3, 4, 5]] specifies 3 feature groups.In this case, possible feature selection results with k_features=2 are [[1], [2] , [[1], [3, 4, 5]] , or [[2], [3, 4, 5]] . Feature groups can be useful for interpretability, for example, if features 3, 4, 5 are one-hot encoded features. (For more details, please read the notes at the bottom of this docstring). New in mlxtend v. 0.21.0. Attributes best_idx_ : array-like, shape = [n_predictions] Feature Indices of the selected feature subsets. best_feature_names_ : array-like, shape = [n_predictions] Feature names of the selected feature subsets. If pandas DataFrames are used in the fit method, the feature names correspond to the column names. Otherwise, the feature names are string representation of the feature array indices. New in v 0.13.0. best_score_ : float Cross validation average score of the selected subset. subsets_ : dict A dictionary of selected feature subsets during the exhaustive selection, where the dictionary keys are the lengths k of these feature subsets. The dictionary values are dictionaries themselves with the following keys: 'feature_idx' (tuple of indices of the feature subset) 'feature_names' (tuple of feature names of the feat. subset) 'cv_scores' (list individual cross-validation scores) 'avg_score' (average cross-validation score) Note that if pandas DataFrames are used in the fit method, the 'feature_names' correspond to the column names. Otherwise, the feature names are string representation of the feature array indices. The 'feature_names' is new in v. 0.13.0. Notes (1) If parameter feature_groups is not None, the number of features is equal to the number of feature groups, i.e. len(feature_groups) . For example, if feature_groups = [[0], [1], [2, 3], [4]] , then the max_features value cannot exceed 4. (2) Although two or more individual features may be considered as one group throughout the feature-selection process, it does not mean the individual features of that group have the same impact on the outcome. For instance, in linear regression, the coefficient of the feature 2 and 3 can be different even if they are considered as one group in feature_groups. (3) If both fixed_features and feature_groups are specified, ensure that each feature group contains the fixed_features selection. E.g., for a 3-feature set fixed_features=[0, 1] and feature_groups=[[0, 1], [2]] is valid; fixed_features=[0, 1] and feature_groups=[[0], [1, 2]] is not valid. (4) In case of KeyboardInterrupt, the dictionary subsets may not be completed. If user is still interested in getting the best score, they can use method `finalize_fit`. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/feature_selection/ExhaustiveFeatureSelector/ Methods finalize_fit() None fit(X, y, groups=None, fit_params) Perform feature selection and learn model from training data. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. New in v 0.13.0: pandas DataFrames are now also accepted as argument for X. y : array-like, shape = [n_samples] Target values. groups : array-like, with shape (n_samples,), optional Group labels for the samples used while splitting the dataset into train/test set. Passed to the fit method of the cross-validator. fit_params : dict of string -> object, optional Parameters to pass to to the fit method of classifier. Returns self : object fit_transform(X, y, groups=None, fit_params) Fit to training data and return the best selected features from X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. New in v 0.13.0: pandas DataFrames are now also accepted as argument for X. y : array-like, shape = [n_samples] Target values. groups : array-like, with shape (n_samples,), optional Group labels for the samples used while splitting the dataset into train/test set. Passed to the fit method of the cross-validator. fit_params : dict of string -> object, optional Parameters to pass to to the fit method of classifier. Returns Feature subset of X, shape={n_samples, k_features} get_metric_dict(confidence_interval=0.95) Return metric dictionary Parameters confidence_interval : float (default: 0.95) A positive float between 0.0 and 1.0 to compute the confidence interval bounds of the CV score averages. Returns Dictionary with items where each dictionary value is a list with the number of iterations (number of feature subsets) as its length. The dictionary keys corresponding to these lists are as follows: 'feature_idx': tuple of the indices of the feature subset 'cv_scores': list with individual CV scores 'avg_score': of CV average scores 'std_dev': standard deviation of the CV score average 'std_err': standard error of the CV score average 'ci_bound': confidence interval bound of the CV score average get_params(deep=True) Get parameters for this estimator. Parameters deep : bool, default=True If True, will return the parameters for this estimator and contained subobjects that are estimators. Returns params : dict Parameter names mapped to their values. set_params( params) Set the parameters of this estimator. The method works on simple estimators as well as on nested objects (such as :class:`~sklearn.pipeline.Pipeline`). The latter have parameters of the form ``<component>__<parameter>`` so that it's possible to update each component of a nested object. Parameters **params : dict Estimator parameters. Returns self : estimator instance Estimator instance. transform(X) Return the best selected features from X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. New in v 0.13.0: pandas DataFrames are now also accepted as argument for X. Returns Feature subset of X, shape={n_samples, k_features} SequentialFeatureSelector SequentialFeatureSelector(estimator, k_features=1, forward=True, floating=False, verbose=0, scoring=None, cv=5, n_jobs=1, pre_dispatch='2 n_jobs', clone_estimator=True, fixed_features=None, feature_groups=None)* Sequential Feature Selection for Classification and Regression. Parameters estimator : scikit-learn classifier or regressor k_features : int or tuple or str (default: 1) Number of features to select, where k_features < the full feature set. New in 0.4.2: A tuple containing a min and max value can be provided, and the SFS will consider return any feature combination between min and max that scored highest in cross-validation. For example, the tuple (1, 4) will return any combination from 1 up to 4 features instead of a fixed number of features k. New in 0.8.0: A string argument \"best\" or \"parsimonious\". If \"best\" is provided, the feature selector will return the feature subset with the best cross-validation performance. If \"parsimonious\" is provided as an argument, the smallest feature subset that is within one standard error of the cross-validation performance will be selected. forward : bool (default: True) Forward selection if True, backward selection otherwise floating : bool (default: False) Adds a conditional exclusion/inclusion if True. verbose : int (default: 0), level of verbosity to use in logging. If 0, no output, if 1 number of features in current set, if 2 detailed logging i ncluding timestamp and cv scores at step. scoring : str, callable, or None (default: None) If None (default), uses 'accuracy' for sklearn classifiers and 'r2' for sklearn regressors. If str, uses a sklearn scoring metric string identifier, for example {accuracy, f1, precision, recall, roc_auc} for classifiers, {'mean_absolute_error', 'mean_squared_error'/'neg_mean_squared_error', 'median_absolute_error', 'r2'} for regressors. If a callable object or function is provided, it has to be conform with sklearn's signature scorer(estimator, X, y) ; see https://scikit-learn.org/stable/modules/generated/sklearn.metrics.make_scorer.html for more information. cv : int (default: 5) Integer or iterable yielding train, test splits. If cv is an integer and estimator is a classifier (or y consists of integer class labels) stratified k-fold. Otherwise regular k-fold cross-validation is performed. No cross-validation if cv is None, False, or 0. n_jobs : int (default: 1) The number of CPUs to use for evaluating different feature subsets in parallel. -1 means 'all CPUs'. pre_dispatch : int, or string (default: '2*n_jobs') Controls the number of jobs that get dispatched during parallel execution if n_jobs > 1 or n_jobs=-1 . Reducing this number can be useful to avoid an explosion of memory consumption when more jobs get dispatched than CPUs can process. This parameter can be: None, in which case all the jobs are immediately created and spawned. Use this for lightweight and fast-running jobs, to avoid delays due to on-demand spawning of the jobs An int, giving the exact number of total jobs that are spawned A string, giving an expression as a function of n_jobs, as in 2*n_jobs clone_estimator : bool (default: True) Clones estimator if True; works with the original estimator instance if False. Set to False if the estimator doesn't implement scikit-learn's set_params and get_params methods. In addition, it is required to set cv=0, and n_jobs=1. fixed_features : tuple (default: None) If not None , the feature indices provided as a tuple will be regarded as fixed by the feature selector. For example, if fixed_features=(1, 3, 7) , the 2nd, 4th, and 8th feature are guaranteed to be present in the solution. Note that if fixed_features is not None , make sure that the number of features to be selected is greater than len(fixed_features) . In other words, ensure that k_features > len(fixed_features) . New in mlxtend v. 0.18.0. feature_groups : list or None (default: None) Optional argument for treating certain features as a group. This means, the features within a group are always selected together, never split. For example, feature_groups=[[1], [2], [3, 4, 5]] specifies 3 feature groups. In this case, possible feature selection results with k_features=2 are [[1], [2] , [[1], [3, 4, 5]] , or [[2], [3, 4, 5]] . Feature groups can be useful for interpretability, for example, if features 3, 4, 5 are one-hot encoded features. (For more details, please read the notes at the bottom of this docstring). New in mlxtend v. 0.21.0. Attributes k_feature_idx_ : array-like, shape = [n_predictions] Feature Indices of the selected feature subsets. k_feature_names_ : array-like, shape = [n_predictions] Feature names of the selected feature subsets. If pandas DataFrames are used in the fit method, the feature names correspond to the column names. Otherwise, the feature names are string representation of the feature array indices. New in v 0.13.0. k_score_ : float Cross validation average score of the selected subset. subsets_ : dict A dictionary of selected feature subsets during the sequential selection, where the dictionary keys are the lengths k of these feature subsets. If the parameter feature_groups is not None, the value of key indicates the number of groups that are selected together. The dictionary values are dictionaries themselves with the following keys: 'feature_idx' (tuple of indices of the feature subset) 'feature_names' (tuple of feature names of the feat. subset) 'cv_scores' (list individual cross-validation scores) 'avg_score' (average cross-validation score) Note that if pandas DataFrames are used in the fit method, the 'feature_names' correspond to the column names. Otherwise, the feature names are string representation of the feature array indices. The 'feature_names' is new in v 0.13.0. Notes (1) If parameter feature_groups is not None, the number of features is equal to the number of feature groups, i.e. len(feature_groups) . For example, if feature_groups = [[0], [1], [2, 3], [4]] , then the max_features value cannot exceed 4. (2) Although two or more individual features may be considered as one group throughout the feature-selection process, it does not mean the individual features of that group have the same impact on the outcome. For instance, in linear regression, the coefficient of the feature 2 and 3 can be different even if they are considered as one group in feature_groups. (3) If both fixed_features and feature_groups are specified, ensure that each feature group contains the fixed_features selection. E.g., for a 3-feature set fixed_features=[0, 1] and feature_groups=[[0, 1], [2]] is valid; fixed_features=[0, 1] and feature_groups=[[0], [1, 2]] is not valid. (4) In case of KeyboardInterrupt, the dictionary subsets may not be completed. If user is still interested in getting the best score, they can use method `finalize_fit`. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/feature_selection/SequentialFeatureSelector/ Methods finalize_fit() None fit(X, y, groups=None, fit_params) Perform feature selection and learn model from training data. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. New in v 0.13.0: pandas DataFrames are now also accepted as argument for X. y : array-like, shape = [n_samples] Target values. New in v 0.13.0: pandas DataFrames are now also accepted as argument for y. groups : array-like, with shape (n_samples,), optional Group labels for the samples used while splitting the dataset into train/test set. Passed to the fit method of the cross-validator. fit_params : various, optional Additional parameters that are being passed to the estimator. For example, sample_weights=weights . Returns self : object fit_transform(X, y, groups=None, fit_params) Fit to training data then reduce X to its most important features. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. New in v 0.13.0: pandas DataFrames are now also accepted as argument for X. y : array-like, shape = [n_samples] Target values. New in v 0.13.0: a pandas Series are now also accepted as argument for y. groups : array-like, with shape (n_samples,), optional Group labels for the samples used while splitting the dataset into train/test set. Passed to the fit method of the cross-validator. fit_params : various, optional Additional parameters that are being passed to the estimator. For example, sample_weights=weights . Returns Reduced feature subset of X, shape={n_samples, k_features} generate_error_message_k_features(name) None get_metric_dict(confidence_interval=0.95) Return metric dictionary Parameters confidence_interval : float (default: 0.95) A positive float between 0.0 and 1.0 to compute the confidence interval bounds of the CV score averages. Returns Dictionary with items where each dictionary value is a list with the number of iterations (number of feature subsets) as its length. The dictionary keys corresponding to these lists are as follows: 'feature_idx': tuple of the indices of the feature subset 'cv_scores': list with individual CV scores 'avg_score': of CV average scores 'std_dev': standard deviation of the CV score average 'std_err': standard error of the CV score average 'ci_bound': confidence interval bound of the CV score average get_params(deep=True) Get parameters for this estimator. Parameters deep : bool, default=True If True, will return the parameters for this estimator and contained subobjects that are estimators. Returns params : dict Parameter names mapped to their values. set_params( params) Set the parameters of this estimator. Valid parameter keys can be listed with get_params() . Returns self transform(X) Reduce X to its most important features. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. New in v 0.13.0: pandas DataFrames are now also accepted as argument for X. Returns Reduced feature subset of X, shape={n_samples, k_features} Properties named_estimators Returns List of named estimator tuples, like [('svc', SVC(...))]","title":"Mlxtend.feature selection"},{"location":"api_subpackages/mlxtend.feature_selection/#columnselector","text":"ColumnSelector(cols=None, drop_axis=False) Object for selecting specific columns from a data set. Parameters cols : array-like (default: None) A list specifying the feature indices to be selected. For example, [1, 4, 5] to select the 2nd, 5th, and 6th feature columns, and ['A','C','D'] to select the name of feature columns A, C and D. If None, returns all columns in the array. drop_axis : bool (default=False) Drops last axis if True and the only one column is selected. This is useful, e.g., when the ColumnSelector is used for selecting only one column and the resulting array should be fed to e.g., a scikit-learn column selector. E.g., instead of returning an array with shape (n_samples, 1), drop_axis=True will return an aray with shape (n_samples,). Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/feature_selection/ColumnSelector/","title":"ColumnSelector"},{"location":"api_subpackages/mlxtend.feature_selection/#methods","text":"fit(X, y=None) Mock method. Does nothing. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] (default: None) Returns self fit_transform(X, y=None) Return a slice of the input array. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] (default: None) Returns X_slice : shape = [n_samples, k_features] Subset of the feature space where k_features <= n_features get_params(deep=True) Get parameters for this estimator. Parameters deep : bool, default=True If True, will return the parameters for this estimator and contained subobjects that are estimators. Returns params : dict Parameter names mapped to their values. set_params( params) Set the parameters of this estimator. The method works on simple estimators as well as on nested objects (such as :class:`~sklearn.pipeline.Pipeline`). The latter have parameters of the form ``<component>__<parameter>`` so that it's possible to update each component of a nested object. Parameters **params : dict Estimator parameters. Returns self : estimator instance Estimator instance. transform(X, y=None) Return a slice of the input array. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] (default: None) Returns X_slice : shape = [n_samples, k_features] Subset of the feature space where k_features <= n_features","title":"Methods"},{"location":"api_subpackages/mlxtend.feature_selection/#exhaustivefeatureselector","text":"ExhaustiveFeatureSelector(estimator, min_features=1, max_features=1, print_progress=True, scoring='accuracy', cv=5, n_jobs=1, pre_dispatch='2 n_jobs', clone_estimator=True, fixed_features=None, feature_groups=None)* Exhaustive Feature Selection for Classification and Regression. (new in v0.4.3) Parameters estimator : scikit-learn classifier or regressor min_features : int (default: 1) Minumum number of features to select max_features : int (default: 1) Maximum number of features to select. If parameter feature_groups is not None, the number of features is equal to the number of feature groups, i.e. len(feature_groups) . For example, if feature_groups = [[0], [1], [2, 3], [4]] , then the max_features value cannot exceed 4. print_progress : bool (default: True) Prints progress as the number of epochs to stderr. scoring : str, (default='accuracy') Scoring metric in {accuracy, f1, precision, recall, roc_auc} for classifiers, {'mean_absolute_error', 'mean_squared_error', 'median_absolute_error', 'r2'} for regressors, or a callable object or function with signature scorer(estimator, X, y) . cv : int (default: 5) Scikit-learn cross-validation generator or int . If estimator is a classifier (or y consists of integer class labels), stratified k-fold is performed, and regular k-fold cross-validation otherwise. No cross-validation if cv is None, False, or 0. n_jobs : int (default: 1) The number of CPUs to use for evaluating different feature subsets in parallel. -1 means 'all CPUs'. pre_dispatch : int, or string (default: '2*n_jobs') Controls the number of jobs that get dispatched during parallel execution if n_jobs > 1 or n_jobs=-1 . Reducing this number can be useful to avoid an explosion of memory consumption when more jobs get dispatched than CPUs can process. This parameter can be: None, in which case all the jobs are immediately created and spawned. Use this for lightweight and fast-running jobs, to avoid delays due to on-demand spawning of the jobs An int, giving the exact number of total jobs that are spawned A string, giving an expression as a function of n_jobs, as in 2*n_jobs clone_estimator : bool (default: True) Clones estimator if True; works with the original estimator instance if False. Set to False if the estimator doesn't implement scikit-learn's set_params and get_params methods. In addition, it is required to set cv=0, and n_jobs=1. fixed_features : tuple (default: None) If not None , the feature indices provided as a tuple will be regarded as fixed by the feature selector. For example, if fixed_features=(1, 3, 7) , the 2nd, 4th, and 8th feature are guaranteed to be present in the solution. Note that if fixed_features is not None , make sure that the number of features to be selected is greater than len(fixed_features) . In other words, ensure that k_features > len(fixed_features) . feature_groups : list or None (default: None) Optional argument for treating certain features as a group. This means, the features within a group are always selected together, never split. For example, feature_groups=[[1], [2], [3, 4, 5]] specifies 3 feature groups.In this case, possible feature selection results with k_features=2 are [[1], [2] , [[1], [3, 4, 5]] , or [[2], [3, 4, 5]] . Feature groups can be useful for interpretability, for example, if features 3, 4, 5 are one-hot encoded features. (For more details, please read the notes at the bottom of this docstring). New in mlxtend v. 0.21.0. Attributes best_idx_ : array-like, shape = [n_predictions] Feature Indices of the selected feature subsets. best_feature_names_ : array-like, shape = [n_predictions] Feature names of the selected feature subsets. If pandas DataFrames are used in the fit method, the feature names correspond to the column names. Otherwise, the feature names are string representation of the feature array indices. New in v 0.13.0. best_score_ : float Cross validation average score of the selected subset. subsets_ : dict A dictionary of selected feature subsets during the exhaustive selection, where the dictionary keys are the lengths k of these feature subsets. The dictionary values are dictionaries themselves with the following keys: 'feature_idx' (tuple of indices of the feature subset) 'feature_names' (tuple of feature names of the feat. subset) 'cv_scores' (list individual cross-validation scores) 'avg_score' (average cross-validation score) Note that if pandas DataFrames are used in the fit method, the 'feature_names' correspond to the column names. Otherwise, the feature names are string representation of the feature array indices. The 'feature_names' is new in v. 0.13.0. Notes (1) If parameter feature_groups is not None, the number of features is equal to the number of feature groups, i.e. len(feature_groups) . For example, if feature_groups = [[0], [1], [2, 3], [4]] , then the max_features value cannot exceed 4. (2) Although two or more individual features may be considered as one group throughout the feature-selection process, it does not mean the individual features of that group have the same impact on the outcome. For instance, in linear regression, the coefficient of the feature 2 and 3 can be different even if they are considered as one group in feature_groups. (3) If both fixed_features and feature_groups are specified, ensure that each feature group contains the fixed_features selection. E.g., for a 3-feature set fixed_features=[0, 1] and feature_groups=[[0, 1], [2]] is valid; fixed_features=[0, 1] and feature_groups=[[0], [1, 2]] is not valid. (4) In case of KeyboardInterrupt, the dictionary subsets may not be completed. If user is still interested in getting the best score, they can use method `finalize_fit`. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/feature_selection/ExhaustiveFeatureSelector/","title":"ExhaustiveFeatureSelector"},{"location":"api_subpackages/mlxtend.feature_selection/#methods_1","text":"finalize_fit() None fit(X, y, groups=None, fit_params) Perform feature selection and learn model from training data. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. New in v 0.13.0: pandas DataFrames are now also accepted as argument for X. y : array-like, shape = [n_samples] Target values. groups : array-like, with shape (n_samples,), optional Group labels for the samples used while splitting the dataset into train/test set. Passed to the fit method of the cross-validator. fit_params : dict of string -> object, optional Parameters to pass to to the fit method of classifier. Returns self : object fit_transform(X, y, groups=None, fit_params) Fit to training data and return the best selected features from X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. New in v 0.13.0: pandas DataFrames are now also accepted as argument for X. y : array-like, shape = [n_samples] Target values. groups : array-like, with shape (n_samples,), optional Group labels for the samples used while splitting the dataset into train/test set. Passed to the fit method of the cross-validator. fit_params : dict of string -> object, optional Parameters to pass to to the fit method of classifier. Returns Feature subset of X, shape={n_samples, k_features} get_metric_dict(confidence_interval=0.95) Return metric dictionary Parameters confidence_interval : float (default: 0.95) A positive float between 0.0 and 1.0 to compute the confidence interval bounds of the CV score averages. Returns Dictionary with items where each dictionary value is a list with the number of iterations (number of feature subsets) as its length. The dictionary keys corresponding to these lists are as follows: 'feature_idx': tuple of the indices of the feature subset 'cv_scores': list with individual CV scores 'avg_score': of CV average scores 'std_dev': standard deviation of the CV score average 'std_err': standard error of the CV score average 'ci_bound': confidence interval bound of the CV score average get_params(deep=True) Get parameters for this estimator. Parameters deep : bool, default=True If True, will return the parameters for this estimator and contained subobjects that are estimators. Returns params : dict Parameter names mapped to their values. set_params( params) Set the parameters of this estimator. The method works on simple estimators as well as on nested objects (such as :class:`~sklearn.pipeline.Pipeline`). The latter have parameters of the form ``<component>__<parameter>`` so that it's possible to update each component of a nested object. Parameters **params : dict Estimator parameters. Returns self : estimator instance Estimator instance. transform(X) Return the best selected features from X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. New in v 0.13.0: pandas DataFrames are now also accepted as argument for X. Returns Feature subset of X, shape={n_samples, k_features}","title":"Methods"},{"location":"api_subpackages/mlxtend.feature_selection/#sequentialfeatureselector","text":"SequentialFeatureSelector(estimator, k_features=1, forward=True, floating=False, verbose=0, scoring=None, cv=5, n_jobs=1, pre_dispatch='2 n_jobs', clone_estimator=True, fixed_features=None, feature_groups=None)* Sequential Feature Selection for Classification and Regression. Parameters estimator : scikit-learn classifier or regressor k_features : int or tuple or str (default: 1) Number of features to select, where k_features < the full feature set. New in 0.4.2: A tuple containing a min and max value can be provided, and the SFS will consider return any feature combination between min and max that scored highest in cross-validation. For example, the tuple (1, 4) will return any combination from 1 up to 4 features instead of a fixed number of features k. New in 0.8.0: A string argument \"best\" or \"parsimonious\". If \"best\" is provided, the feature selector will return the feature subset with the best cross-validation performance. If \"parsimonious\" is provided as an argument, the smallest feature subset that is within one standard error of the cross-validation performance will be selected. forward : bool (default: True) Forward selection if True, backward selection otherwise floating : bool (default: False) Adds a conditional exclusion/inclusion if True. verbose : int (default: 0), level of verbosity to use in logging. If 0, no output, if 1 number of features in current set, if 2 detailed logging i ncluding timestamp and cv scores at step. scoring : str, callable, or None (default: None) If None (default), uses 'accuracy' for sklearn classifiers and 'r2' for sklearn regressors. If str, uses a sklearn scoring metric string identifier, for example {accuracy, f1, precision, recall, roc_auc} for classifiers, {'mean_absolute_error', 'mean_squared_error'/'neg_mean_squared_error', 'median_absolute_error', 'r2'} for regressors. If a callable object or function is provided, it has to be conform with sklearn's signature scorer(estimator, X, y) ; see https://scikit-learn.org/stable/modules/generated/sklearn.metrics.make_scorer.html for more information. cv : int (default: 5) Integer or iterable yielding train, test splits. If cv is an integer and estimator is a classifier (or y consists of integer class labels) stratified k-fold. Otherwise regular k-fold cross-validation is performed. No cross-validation if cv is None, False, or 0. n_jobs : int (default: 1) The number of CPUs to use for evaluating different feature subsets in parallel. -1 means 'all CPUs'. pre_dispatch : int, or string (default: '2*n_jobs') Controls the number of jobs that get dispatched during parallel execution if n_jobs > 1 or n_jobs=-1 . Reducing this number can be useful to avoid an explosion of memory consumption when more jobs get dispatched than CPUs can process. This parameter can be: None, in which case all the jobs are immediately created and spawned. Use this for lightweight and fast-running jobs, to avoid delays due to on-demand spawning of the jobs An int, giving the exact number of total jobs that are spawned A string, giving an expression as a function of n_jobs, as in 2*n_jobs clone_estimator : bool (default: True) Clones estimator if True; works with the original estimator instance if False. Set to False if the estimator doesn't implement scikit-learn's set_params and get_params methods. In addition, it is required to set cv=0, and n_jobs=1. fixed_features : tuple (default: None) If not None , the feature indices provided as a tuple will be regarded as fixed by the feature selector. For example, if fixed_features=(1, 3, 7) , the 2nd, 4th, and 8th feature are guaranteed to be present in the solution. Note that if fixed_features is not None , make sure that the number of features to be selected is greater than len(fixed_features) . In other words, ensure that k_features > len(fixed_features) . New in mlxtend v. 0.18.0. feature_groups : list or None (default: None) Optional argument for treating certain features as a group. This means, the features within a group are always selected together, never split. For example, feature_groups=[[1], [2], [3, 4, 5]] specifies 3 feature groups. In this case, possible feature selection results with k_features=2 are [[1], [2] , [[1], [3, 4, 5]] , or [[2], [3, 4, 5]] . Feature groups can be useful for interpretability, for example, if features 3, 4, 5 are one-hot encoded features. (For more details, please read the notes at the bottom of this docstring). New in mlxtend v. 0.21.0. Attributes k_feature_idx_ : array-like, shape = [n_predictions] Feature Indices of the selected feature subsets. k_feature_names_ : array-like, shape = [n_predictions] Feature names of the selected feature subsets. If pandas DataFrames are used in the fit method, the feature names correspond to the column names. Otherwise, the feature names are string representation of the feature array indices. New in v 0.13.0. k_score_ : float Cross validation average score of the selected subset. subsets_ : dict A dictionary of selected feature subsets during the sequential selection, where the dictionary keys are the lengths k of these feature subsets. If the parameter feature_groups is not None, the value of key indicates the number of groups that are selected together. The dictionary values are dictionaries themselves with the following keys: 'feature_idx' (tuple of indices of the feature subset) 'feature_names' (tuple of feature names of the feat. subset) 'cv_scores' (list individual cross-validation scores) 'avg_score' (average cross-validation score) Note that if pandas DataFrames are used in the fit method, the 'feature_names' correspond to the column names. Otherwise, the feature names are string representation of the feature array indices. The 'feature_names' is new in v 0.13.0. Notes (1) If parameter feature_groups is not None, the number of features is equal to the number of feature groups, i.e. len(feature_groups) . For example, if feature_groups = [[0], [1], [2, 3], [4]] , then the max_features value cannot exceed 4. (2) Although two or more individual features may be considered as one group throughout the feature-selection process, it does not mean the individual features of that group have the same impact on the outcome. For instance, in linear regression, the coefficient of the feature 2 and 3 can be different even if they are considered as one group in feature_groups. (3) If both fixed_features and feature_groups are specified, ensure that each feature group contains the fixed_features selection. E.g., for a 3-feature set fixed_features=[0, 1] and feature_groups=[[0, 1], [2]] is valid; fixed_features=[0, 1] and feature_groups=[[0], [1, 2]] is not valid. (4) In case of KeyboardInterrupt, the dictionary subsets may not be completed. If user is still interested in getting the best score, they can use method `finalize_fit`. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/feature_selection/SequentialFeatureSelector/","title":"SequentialFeatureSelector"},{"location":"api_subpackages/mlxtend.feature_selection/#methods_2","text":"finalize_fit() None fit(X, y, groups=None, fit_params) Perform feature selection and learn model from training data. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. New in v 0.13.0: pandas DataFrames are now also accepted as argument for X. y : array-like, shape = [n_samples] Target values. New in v 0.13.0: pandas DataFrames are now also accepted as argument for y. groups : array-like, with shape (n_samples,), optional Group labels for the samples used while splitting the dataset into train/test set. Passed to the fit method of the cross-validator. fit_params : various, optional Additional parameters that are being passed to the estimator. For example, sample_weights=weights . Returns self : object fit_transform(X, y, groups=None, fit_params) Fit to training data then reduce X to its most important features. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. New in v 0.13.0: pandas DataFrames are now also accepted as argument for X. y : array-like, shape = [n_samples] Target values. New in v 0.13.0: a pandas Series are now also accepted as argument for y. groups : array-like, with shape (n_samples,), optional Group labels for the samples used while splitting the dataset into train/test set. Passed to the fit method of the cross-validator. fit_params : various, optional Additional parameters that are being passed to the estimator. For example, sample_weights=weights . Returns Reduced feature subset of X, shape={n_samples, k_features} generate_error_message_k_features(name) None get_metric_dict(confidence_interval=0.95) Return metric dictionary Parameters confidence_interval : float (default: 0.95) A positive float between 0.0 and 1.0 to compute the confidence interval bounds of the CV score averages. Returns Dictionary with items where each dictionary value is a list with the number of iterations (number of feature subsets) as its length. The dictionary keys corresponding to these lists are as follows: 'feature_idx': tuple of the indices of the feature subset 'cv_scores': list with individual CV scores 'avg_score': of CV average scores 'std_dev': standard deviation of the CV score average 'std_err': standard error of the CV score average 'ci_bound': confidence interval bound of the CV score average get_params(deep=True) Get parameters for this estimator. Parameters deep : bool, default=True If True, will return the parameters for this estimator and contained subobjects that are estimators. Returns params : dict Parameter names mapped to their values. set_params( params) Set the parameters of this estimator. Valid parameter keys can be listed with get_params() . Returns self transform(X) Reduce X to its most important features. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. New in v 0.13.0: pandas DataFrames are now also accepted as argument for X. Returns Reduced feature subset of X, shape={n_samples, k_features}","title":"Methods"},{"location":"api_subpackages/mlxtend.feature_selection/#properties","text":"named_estimators Returns List of named estimator tuples, like [('svc', SVC(...))]","title":"Properties"},{"location":"api_subpackages/mlxtend.file_io/","text":"mlxtend version: 0.23.1 find_filegroups find_filegroups(paths, substring='', extensions=None, validity_check=True, ignore_invisible=True, rstrip='', ignore_substring=None) Find and collect files from different directories in a python dictionary. Parameters paths : list Paths of the directories to be searched. Dictionary keys are build from the first directory. substring : str (default: '') Substring that all files have to contain to be considered. extensions : list (default: None) None or list of allowed file extensions for each path. If provided, the number of extensions must match the number of paths . validity_check : bool (default: None) If True , checks if all dictionary values have the same number of file paths. Prints a warning and returns an empty dictionary if the validity check failed. ignore_invisible : bool (default: True) If True , ignores invisible files (i.e., files starting with a period). rstrip : str (default: '') If provided, strips characters from right side of the file base names after splitting the extension. Useful to trim different filenames to a common stem. E.g,. \"abc_d.txt\" and \"abc_d_.csv\" would share the stem \"abc_d\" if rstrip is set to \"_\". ignore_substring : str (default: None) Ignores files that contain the specified substring. Returns groups : dict Dictionary of files paths. Keys are the file names found in the first directory listed in paths (without file extension). Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/file_io/find_filegroups/ find_files find_files(substring, path, recursive=False, check_ext=None, ignore_invisible=True, ignore_substring=None) Find files in a directory based on substring matching. Parameters substring : str Substring of the file to be matched. path : str Path where to look. recursive : bool If true, searches subdirectories recursively. check_ext : str If string (e.g., '.txt'), only returns files that match the specified file extension. ignore_invisible : bool If True , ignores invisible files (i.e., files starting with a period). ignore_substring : str Ignores files that contain the specified substring. Returns results : list List of the matched files. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/file_io/find_files/","title":"Mlxtend.file io"},{"location":"api_subpackages/mlxtend.file_io/#find_filegroups","text":"find_filegroups(paths, substring='', extensions=None, validity_check=True, ignore_invisible=True, rstrip='', ignore_substring=None) Find and collect files from different directories in a python dictionary. Parameters paths : list Paths of the directories to be searched. Dictionary keys are build from the first directory. substring : str (default: '') Substring that all files have to contain to be considered. extensions : list (default: None) None or list of allowed file extensions for each path. If provided, the number of extensions must match the number of paths . validity_check : bool (default: None) If True , checks if all dictionary values have the same number of file paths. Prints a warning and returns an empty dictionary if the validity check failed. ignore_invisible : bool (default: True) If True , ignores invisible files (i.e., files starting with a period). rstrip : str (default: '') If provided, strips characters from right side of the file base names after splitting the extension. Useful to trim different filenames to a common stem. E.g,. \"abc_d.txt\" and \"abc_d_.csv\" would share the stem \"abc_d\" if rstrip is set to \"_\". ignore_substring : str (default: None) Ignores files that contain the specified substring. Returns groups : dict Dictionary of files paths. Keys are the file names found in the first directory listed in paths (without file extension). Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/file_io/find_filegroups/","title":"find_filegroups"},{"location":"api_subpackages/mlxtend.file_io/#find_files","text":"find_files(substring, path, recursive=False, check_ext=None, ignore_invisible=True, ignore_substring=None) Find files in a directory based on substring matching. Parameters substring : str Substring of the file to be matched. path : str Path where to look. recursive : bool If true, searches subdirectories recursively. check_ext : str If string (e.g., '.txt'), only returns files that match the specified file extension. ignore_invisible : bool If True , ignores invisible files (i.e., files starting with a period). ignore_substring : str Ignores files that contain the specified substring. Returns results : list List of the matched files. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/file_io/find_files/","title":"find_files"},{"location":"api_subpackages/mlxtend.frequent_patterns/","text":"mlxtend version: 0.23.1 apriori apriori(df, min_support=0.5, use_colnames=False, max_len=None, verbose=0, low_memory=False) Get frequent itemsets from a one-hot DataFrame Parameters df : pandas DataFrame pandas DataFrame the encoded format. Also supports DataFrames with sparse data; for more info, please see (https://pandas.pydata.org/pandas-docs/stable/ user_guide/sparse.html#sparse-data-structures) Please note that the old pandas SparseDataFrame format is no longer supported in mlxtend >= 0.17.2. The allowed values are either 0/1 or True/False. For example, Apple Bananas Beer Chicken Milk Rice 0 True False True True False True 1 True False True False False True 2 True False True False False False 3 True True False False False False 4 False False True True True True 5 False False True False True True 6 False False True False True False 7 True True False False False False min_support : float (default: 0.5) A float between 0 and 1 for minumum support of the itemsets returned. The support is computed as the fraction transactions_where_item(s)_occur / total_transactions . use_colnames : bool (default: False) If True , uses the DataFrames' column names in the returned DataFrame instead of column indices. max_len : int (default: None) Maximum length of the itemsets generated. If None (default) all possible itemsets lengths (under the apriori condition) are evaluated. verbose : int (default: 0) Shows the number of iterations if >= 1 and low_memory is True . If =1 and low_memory is False , shows the number of combinations. low_memory : bool (default: False) If True , uses an iterator to search for combinations above min_support . Note that while low_memory=True should only be used for large dataset if memory resources are limited, because this implementation is approx. 3-6x slower than the default. Returns pandas DataFrame with columns ['support', 'itemsets'] of all itemsets that are >= min_support and < than max_len (if max_len is not None). Each itemset in the 'itemsets' column is of type frozenset , which is a Python built-in type that behaves similarly to sets except that it is immutable (For more info, see https://docs.python.org/3.6/library/stdtypes.html#frozenset). Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/frequent_patterns/apriori/ association_rules association_rules(df, metric='confidence', min_threshold=0.8, support_only=False) Generates a DataFrame of association rules including the metrics 'score', 'confidence', and 'lift' Parameters df : pandas DataFrame pandas DataFrame of frequent itemsets with columns ['support', 'itemsets'] metric : string (default: 'confidence') Metric to evaluate if a rule is of interest. Automatically set to 'support' if support_only=True . Otherwise, supported metrics are 'support', 'confidence', 'lift', 'leverage', 'conviction' and 'zhangs_metric' These metrics are computed as follows: - support(A->C) = support(A+C) [aka 'support'], range: [0, 1] - confidence(A->C) = support(A+C) / support(A), range: [0, 1] - lift(A->C) = confidence(A->C) / support(C), range: [0, inf] - leverage(A->C) = support(A->C) - support(A)*support(C), range: [-1, 1] - conviction = [1 - support(C)] / [1 - confidence(A->C)], range: [0, inf] - zhangs_metric(A->C) = leverage(A->C) / max(support(A->C)*(1-support(A)), support(A)*(support(C)-support(A->C))) range: [-1,1] min_threshold : float (default: 0.8) Minimal threshold for the evaluation metric, via the metric parameter, to decide whether a candidate rule is of interest. support_only : bool (default: False) Only computes the rule support and fills the other metric columns with NaNs. This is useful if: a) the input DataFrame is incomplete, e.g., does not contain support values for all rule antecedents and consequents b) you simply want to speed up the computation because you don't need the other metrics. Returns pandas DataFrame with columns \"antecedents\" and \"consequents\" that store itemsets, plus the scoring metric columns: \"antecedent support\", \"consequent support\", \"support\", \"confidence\", \"lift\", \"leverage\", \"conviction\" of all rules for which metric(rule) >= min_threshold. Each entry in the \"antecedents\" and \"consequents\" columns are of type frozenset , which is a Python built-in type that behaves similarly to sets except that it is immutable (For more info, see https://docs.python.org/3.6/library/stdtypes.html#frozenset). Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/frequent_patterns/association_rules/ fpgrowth fpgrowth(df, min_support=0.5, use_colnames=False, max_len=None, verbose=0) Get frequent itemsets from a one-hot DataFrame Parameters df : pandas DataFrame pandas DataFrame the encoded format. Also supports DataFrames with sparse data; for more info, please see https://pandas.pydata.org/pandas-docs/stable/user_guide/sparse.html#sparse-data-structures. Please note that the old pandas SparseDataFrame format is no longer supported in mlxtend >= 0.17.2. The allowed values are either 0/1 or True/False. For example, Apple Bananas Beer Chicken Milk Rice 0 True False True True False True 1 True False True False False True 2 True False True False False False 3 True True False False False False 4 False False True True True True 5 False False True False True True 6 False False True False True False 7 True True False False False False min_support : float (default: 0.5) A float between 0 and 1 for minimum support of the itemsets returned. The support is computed as the fraction transactions_where_item(s)_occur / total_transactions. use_colnames : bool (default: False) If true, uses the DataFrames' column names in the returned DataFrame instead of column indices. max_len : int (default: None) Maximum length of the itemsets generated. If None (default) all possible itemsets lengths are evaluated. verbose : int (default: 0) Shows the stages of conditional tree generation. Returns pandas DataFrame with columns ['support', 'itemsets'] of all itemsets that are >= min_support and < than max_len (if max_len is not None). Each itemset in the 'itemsets' column is of type frozenset , which is a Python built-in type that behaves similarly to sets except that it is immutable (For more info, see https://docs.python.org/3.6/library/stdtypes.html#frozenset). Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/frequent_patterns/fpgrowth/ fpmax fpmax(df, min_support=0.5, use_colnames=False, max_len=None, verbose=0) Get maximal frequent itemsets from a one-hot DataFrame Parameters df : pandas DataFrame pandas DataFrame the encoded format. Also supports DataFrames with sparse data; for more info, please see (https://pandas.pydata.org/pandas-docs/stable/ user_guide/sparse.html#sparse-data-structures) Please note that the old pandas SparseDataFrame format is no longer supported in mlxtend >= 0.17.2. The allowed values are either 0/1 or True/False. For example, Apple Bananas Beer Chicken Milk Rice 0 True False True True False True 1 True False True False False True 2 True False True False False False 3 True True False False False False 4 False False True True True True 5 False False True False True True 6 False False True False True False 7 True True False False False False min_support : float (default: 0.5) A float between 0 and 1 for minimum support of the itemsets returned. The support is computed as the fraction transactions_where_item(s)_occur / total_transactions. use_colnames : bool (default: False) If true, uses the DataFrames' column names in the returned DataFrame instead of column indices. max_len : int (default: None) Given the set of all maximal itemsets, return those that are less than max_len . If None (default) all possible itemsets lengths are evaluated. verbose : int (default: 0) Shows the stages of conditional tree generation. Returns pandas DataFrame with columns ['support', 'itemsets'] of all maximal itemsets that are >= min_support and < than max_len (if max_len is not None). Each itemset in the 'itemsets' column is of type frozenset , which is a Python built-in type that behaves similarly to sets except that it is immutable (For more info, see https://docs.python.org/3.6/library/stdtypes.html#frozenset). Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/frequent_patterns/fpmax/ hmine hmine(df, min_support=0.5, use_colnames=False, max_len=None, verbose=0) -> pandas.core.frame.DataFrame Get frequent itemsets from a one-hot DataFrame Parameters df : pandas DataFrame pandas DataFrame the encoded format. Also supports DataFrames with sparse data; for more info, please see https://pandas.pydata.org/pandas-docs/stable/user_guide/sparse.html#sparse-data-structures. Please note that the old pandas SparseDataFrame format is no longer supported in mlxtend >= 0.17.2. The allowed values are either 0/1 or True/False. For example, Apple Bananas Beer Chicken Milk Rice 0 True False True True False True 1 True False True False False True 2 True False True False False False 3 True True False False False False 4 False False True True True True 5 False False True False True True 6 False False True False True False 7 True True False False False False min_support : float (default: 0.5) A float between 0 and 1 for minimum support of the itemsets returned. The support is computed as the fraction transactions_where_item(s)_occur / total_transactions. use_colnames : bool (default: False) If true, uses the DataFrames' column names in the returned DataFrame instead of column indices. max_len : int (default: None) Maximum length of the itemsets generated. If None (default) all possible itemsets lengths are evaluated. verbose : int (default: 0) Shows the stages of conditional tree generation. Returns pandas DataFrame with columns ['support', 'itemsets'] of all itemsets that are >= min_support and < than max_len (if max_len is not None). Each itemset in the 'itemsets' column is of type frozenset , which is a Python built-in type that behaves similarly to sets except that it is immutable (For more info, see https://docs.python.org/3.6/library/stdtypes.html#frozenset). Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/frequent_patterns/hmine/","title":"Mlxtend.frequent patterns"},{"location":"api_subpackages/mlxtend.frequent_patterns/#apriori","text":"apriori(df, min_support=0.5, use_colnames=False, max_len=None, verbose=0, low_memory=False) Get frequent itemsets from a one-hot DataFrame Parameters df : pandas DataFrame pandas DataFrame the encoded format. Also supports DataFrames with sparse data; for more info, please see (https://pandas.pydata.org/pandas-docs/stable/ user_guide/sparse.html#sparse-data-structures) Please note that the old pandas SparseDataFrame format is no longer supported in mlxtend >= 0.17.2. The allowed values are either 0/1 or True/False. For example, Apple Bananas Beer Chicken Milk Rice 0 True False True True False True 1 True False True False False True 2 True False True False False False 3 True True False False False False 4 False False True True True True 5 False False True False True True 6 False False True False True False 7 True True False False False False min_support : float (default: 0.5) A float between 0 and 1 for minumum support of the itemsets returned. The support is computed as the fraction transactions_where_item(s)_occur / total_transactions . use_colnames : bool (default: False) If True , uses the DataFrames' column names in the returned DataFrame instead of column indices. max_len : int (default: None) Maximum length of the itemsets generated. If None (default) all possible itemsets lengths (under the apriori condition) are evaluated. verbose : int (default: 0) Shows the number of iterations if >= 1 and low_memory is True . If =1 and low_memory is False , shows the number of combinations. low_memory : bool (default: False) If True , uses an iterator to search for combinations above min_support . Note that while low_memory=True should only be used for large dataset if memory resources are limited, because this implementation is approx. 3-6x slower than the default. Returns pandas DataFrame with columns ['support', 'itemsets'] of all itemsets that are >= min_support and < than max_len (if max_len is not None). Each itemset in the 'itemsets' column is of type frozenset , which is a Python built-in type that behaves similarly to sets except that it is immutable (For more info, see https://docs.python.org/3.6/library/stdtypes.html#frozenset). Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/frequent_patterns/apriori/","title":"apriori"},{"location":"api_subpackages/mlxtend.frequent_patterns/#association_rules","text":"association_rules(df, metric='confidence', min_threshold=0.8, support_only=False) Generates a DataFrame of association rules including the metrics 'score', 'confidence', and 'lift' Parameters df : pandas DataFrame pandas DataFrame of frequent itemsets with columns ['support', 'itemsets'] metric : string (default: 'confidence') Metric to evaluate if a rule is of interest. Automatically set to 'support' if support_only=True . Otherwise, supported metrics are 'support', 'confidence', 'lift', 'leverage', 'conviction' and 'zhangs_metric' These metrics are computed as follows: - support(A->C) = support(A+C) [aka 'support'], range: [0, 1] - confidence(A->C) = support(A+C) / support(A), range: [0, 1] - lift(A->C) = confidence(A->C) / support(C), range: [0, inf] - leverage(A->C) = support(A->C) - support(A)*support(C), range: [-1, 1] - conviction = [1 - support(C)] / [1 - confidence(A->C)], range: [0, inf] - zhangs_metric(A->C) = leverage(A->C) / max(support(A->C)*(1-support(A)), support(A)*(support(C)-support(A->C))) range: [-1,1] min_threshold : float (default: 0.8) Minimal threshold for the evaluation metric, via the metric parameter, to decide whether a candidate rule is of interest. support_only : bool (default: False) Only computes the rule support and fills the other metric columns with NaNs. This is useful if: a) the input DataFrame is incomplete, e.g., does not contain support values for all rule antecedents and consequents b) you simply want to speed up the computation because you don't need the other metrics. Returns pandas DataFrame with columns \"antecedents\" and \"consequents\" that store itemsets, plus the scoring metric columns: \"antecedent support\", \"consequent support\", \"support\", \"confidence\", \"lift\", \"leverage\", \"conviction\" of all rules for which metric(rule) >= min_threshold. Each entry in the \"antecedents\" and \"consequents\" columns are of type frozenset , which is a Python built-in type that behaves similarly to sets except that it is immutable (For more info, see https://docs.python.org/3.6/library/stdtypes.html#frozenset). Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/frequent_patterns/association_rules/","title":"association_rules"},{"location":"api_subpackages/mlxtend.frequent_patterns/#fpgrowth","text":"fpgrowth(df, min_support=0.5, use_colnames=False, max_len=None, verbose=0) Get frequent itemsets from a one-hot DataFrame Parameters df : pandas DataFrame pandas DataFrame the encoded format. Also supports DataFrames with sparse data; for more info, please see https://pandas.pydata.org/pandas-docs/stable/user_guide/sparse.html#sparse-data-structures. Please note that the old pandas SparseDataFrame format is no longer supported in mlxtend >= 0.17.2. The allowed values are either 0/1 or True/False. For example, Apple Bananas Beer Chicken Milk Rice 0 True False True True False True 1 True False True False False True 2 True False True False False False 3 True True False False False False 4 False False True True True True 5 False False True False True True 6 False False True False True False 7 True True False False False False min_support : float (default: 0.5) A float between 0 and 1 for minimum support of the itemsets returned. The support is computed as the fraction transactions_where_item(s)_occur / total_transactions. use_colnames : bool (default: False) If true, uses the DataFrames' column names in the returned DataFrame instead of column indices. max_len : int (default: None) Maximum length of the itemsets generated. If None (default) all possible itemsets lengths are evaluated. verbose : int (default: 0) Shows the stages of conditional tree generation. Returns pandas DataFrame with columns ['support', 'itemsets'] of all itemsets that are >= min_support and < than max_len (if max_len is not None). Each itemset in the 'itemsets' column is of type frozenset , which is a Python built-in type that behaves similarly to sets except that it is immutable (For more info, see https://docs.python.org/3.6/library/stdtypes.html#frozenset). Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/frequent_patterns/fpgrowth/","title":"fpgrowth"},{"location":"api_subpackages/mlxtend.frequent_patterns/#fpmax","text":"fpmax(df, min_support=0.5, use_colnames=False, max_len=None, verbose=0) Get maximal frequent itemsets from a one-hot DataFrame Parameters df : pandas DataFrame pandas DataFrame the encoded format. Also supports DataFrames with sparse data; for more info, please see (https://pandas.pydata.org/pandas-docs/stable/ user_guide/sparse.html#sparse-data-structures) Please note that the old pandas SparseDataFrame format is no longer supported in mlxtend >= 0.17.2. The allowed values are either 0/1 or True/False. For example, Apple Bananas Beer Chicken Milk Rice 0 True False True True False True 1 True False True False False True 2 True False True False False False 3 True True False False False False 4 False False True True True True 5 False False True False True True 6 False False True False True False 7 True True False False False False min_support : float (default: 0.5) A float between 0 and 1 for minimum support of the itemsets returned. The support is computed as the fraction transactions_where_item(s)_occur / total_transactions. use_colnames : bool (default: False) If true, uses the DataFrames' column names in the returned DataFrame instead of column indices. max_len : int (default: None) Given the set of all maximal itemsets, return those that are less than max_len . If None (default) all possible itemsets lengths are evaluated. verbose : int (default: 0) Shows the stages of conditional tree generation. Returns pandas DataFrame with columns ['support', 'itemsets'] of all maximal itemsets that are >= min_support and < than max_len (if max_len is not None). Each itemset in the 'itemsets' column is of type frozenset , which is a Python built-in type that behaves similarly to sets except that it is immutable (For more info, see https://docs.python.org/3.6/library/stdtypes.html#frozenset). Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/frequent_patterns/fpmax/","title":"fpmax"},{"location":"api_subpackages/mlxtend.frequent_patterns/#hmine","text":"hmine(df, min_support=0.5, use_colnames=False, max_len=None, verbose=0) -> pandas.core.frame.DataFrame Get frequent itemsets from a one-hot DataFrame Parameters df : pandas DataFrame pandas DataFrame the encoded format. Also supports DataFrames with sparse data; for more info, please see https://pandas.pydata.org/pandas-docs/stable/user_guide/sparse.html#sparse-data-structures. Please note that the old pandas SparseDataFrame format is no longer supported in mlxtend >= 0.17.2. The allowed values are either 0/1 or True/False. For example, Apple Bananas Beer Chicken Milk Rice 0 True False True True False True 1 True False True False False True 2 True False True False False False 3 True True False False False False 4 False False True True True True 5 False False True False True True 6 False False True False True False 7 True True False False False False min_support : float (default: 0.5) A float between 0 and 1 for minimum support of the itemsets returned. The support is computed as the fraction transactions_where_item(s)_occur / total_transactions. use_colnames : bool (default: False) If true, uses the DataFrames' column names in the returned DataFrame instead of column indices. max_len : int (default: None) Maximum length of the itemsets generated. If None (default) all possible itemsets lengths are evaluated. verbose : int (default: 0) Shows the stages of conditional tree generation. Returns pandas DataFrame with columns ['support', 'itemsets'] of all itemsets that are >= min_support and < than max_len (if max_len is not None). Each itemset in the 'itemsets' column is of type frozenset , which is a Python built-in type that behaves similarly to sets except that it is immutable (For more info, see https://docs.python.org/3.6/library/stdtypes.html#frozenset). Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/frequent_patterns/hmine/","title":"hmine"},{"location":"api_subpackages/mlxtend.image/","text":"mlxtend version: 0.23.1 EyepadAlign EyepadAlign(verbose=0) Class to align/transform face images to facial landmarks, based on eye alignment. 1. A scaling factor is computed based on distance between the left and right eye, such that the transformed face image will have the same eye distance as a reference face image. 2. A transformation is performed based on the eyes' center point. to align the face based on the reference eye location. 3. Finally, the transformed image is padded with zeros to match the desired final image size. Parameters verbose : int (default=0) Verbose level to display the progress bar and log messages. Setting verbose=1 will print a progress bar upon calling fit_directory . Attributes target_landmarks_ : target landmarks to transform new face images to. Depending on the chosen fit parameters, it can be either (1) assigned to pre-fit shapes, (2) computed from a single face image (3) computed as the mean of face landmarks from all face images in a file directory of face images. eye_distance_ : the distance between left and right eyes in the target landmarks. target_height_ : the height of the transformed output image. target_width_ : the width of the transformed output image. Returns self : object Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/image/eyepad_align/ Methods fit_directory(target_img_dir, target_height, target_width, file_extension='.jpg', pre_check=True) Calculates the average landmarks for all face images in a directory which will then be set as the target landmark set. Arguments target_img_dir : str Directory containing the images target_height : int Expected image height of the images in the directory target_width : int Expected image width of the images in the directory file_extension str (default='.jpg'): File extension of the image files. pre_check Bool (default=True): Checks that each image has the dimensions specificed via target_height and target_width on the whole directory first to identify potential issues that are recommended to be fixed before proceeding. Raises a warning for each image if dimensions differ from the ones specified and expected. Returns self : object fit_image(target_image) Derives facial landmarks from a target image. Arguments target_image : uint8 numpy.array, shape=[width, height, channels] NumPy array representation of the image data. Returns self : object fit_values(target_landmarks, target_width, target_height) Used for determining the eye location from pre-defined landmark arrays, eliminating the need for re-computing the average landmarks on a target image or image directory. Arguments target_landmarks : np.array, shape=(height, width) NumPy array containing the locations of the facial landmarks as determined by mlxtend.image.extract_face_landmarks target_height : int image height target_width : int image width Returns self : object transform(img) transforms a single face image (img) to the target landmarks based on the location of the eyes by scaling, translation and cropping (if needed): (1) Scaling the image so that the distance of the two eyes in the given image (img) matches the distance of the two eyes in the target landmarks. (2) Translation is performed based on the middle point between the two eyes. Arguments img : np.array, shape=(height, width, channels) Input image to be transformed. Returns self : object extract_face_landmarks extract_face_landmarks(img, return_dtype= ) Function to extract face landmarks. Note that this function requires an installation of the Python version of the library \"dlib\": https://dlib.net Parameters img : array, shape = [h, w, ?] Numpy array of a face image or imageio.core.util.Array. E.g., img = imageio.core.util.Array(ary) Supported shapes are - 3D tensors with 1 or more color channels, for example, RGB: [h, w, 3] - 2D tensors without color channel, for example, Grayscale: [h, w] return_dtype: the return data-type of the array, default: np.int32. Returns landmarks : numpy.ndarray, shape = [68, 2] A numpy array, where each row contains a landmark/point x-y coordinates. Return None if no face is detected by Dlib. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/image/extract_face_landmarks/","title":"Mlxtend.image"},{"location":"api_subpackages/mlxtend.image/#eyepadalign","text":"EyepadAlign(verbose=0) Class to align/transform face images to facial landmarks, based on eye alignment. 1. A scaling factor is computed based on distance between the left and right eye, such that the transformed face image will have the same eye distance as a reference face image. 2. A transformation is performed based on the eyes' center point. to align the face based on the reference eye location. 3. Finally, the transformed image is padded with zeros to match the desired final image size. Parameters verbose : int (default=0) Verbose level to display the progress bar and log messages. Setting verbose=1 will print a progress bar upon calling fit_directory . Attributes target_landmarks_ : target landmarks to transform new face images to. Depending on the chosen fit parameters, it can be either (1) assigned to pre-fit shapes, (2) computed from a single face image (3) computed as the mean of face landmarks from all face images in a file directory of face images. eye_distance_ : the distance between left and right eyes in the target landmarks. target_height_ : the height of the transformed output image. target_width_ : the width of the transformed output image. Returns self : object Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/image/eyepad_align/","title":"EyepadAlign"},{"location":"api_subpackages/mlxtend.image/#methods","text":"fit_directory(target_img_dir, target_height, target_width, file_extension='.jpg', pre_check=True) Calculates the average landmarks for all face images in a directory which will then be set as the target landmark set. Arguments target_img_dir : str Directory containing the images target_height : int Expected image height of the images in the directory target_width : int Expected image width of the images in the directory file_extension str (default='.jpg'): File extension of the image files. pre_check Bool (default=True): Checks that each image has the dimensions specificed via target_height and target_width on the whole directory first to identify potential issues that are recommended to be fixed before proceeding. Raises a warning for each image if dimensions differ from the ones specified and expected. Returns self : object fit_image(target_image) Derives facial landmarks from a target image. Arguments target_image : uint8 numpy.array, shape=[width, height, channels] NumPy array representation of the image data. Returns self : object fit_values(target_landmarks, target_width, target_height) Used for determining the eye location from pre-defined landmark arrays, eliminating the need for re-computing the average landmarks on a target image or image directory. Arguments target_landmarks : np.array, shape=(height, width) NumPy array containing the locations of the facial landmarks as determined by mlxtend.image.extract_face_landmarks target_height : int image height target_width : int image width Returns self : object transform(img) transforms a single face image (img) to the target landmarks based on the location of the eyes by scaling, translation and cropping (if needed): (1) Scaling the image so that the distance of the two eyes in the given image (img) matches the distance of the two eyes in the target landmarks. (2) Translation is performed based on the middle point between the two eyes. Arguments img : np.array, shape=(height, width, channels) Input image to be transformed. Returns self : object","title":"Methods"},{"location":"api_subpackages/mlxtend.image/#extract_face_landmarks","text":"extract_face_landmarks(img, return_dtype= ) Function to extract face landmarks. Note that this function requires an installation of the Python version of the library \"dlib\": https://dlib.net Parameters img : array, shape = [h, w, ?] Numpy array of a face image or imageio.core.util.Array. E.g., img = imageio.core.util.Array(ary) Supported shapes are - 3D tensors with 1 or more color channels, for example, RGB: [h, w, 3] - 2D tensors without color channel, for example, Grayscale: [h, w] return_dtype: the return data-type of the array, default: np.int32. Returns landmarks : numpy.ndarray, shape = [68, 2] A numpy array, where each row contains a landmark/point x-y coordinates. Return None if no face is detected by Dlib. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/image/extract_face_landmarks/","title":"extract_face_landmarks"},{"location":"api_subpackages/mlxtend.math/","text":"mlxtend version: 0.23.1 factorial factorial(n) None num_combinations num_combinations(n, k, with_replacement=False) Function to calculate the number of possible combinations. Parameters n : int Total number of items. k : int Number of elements of the target itemset. with_replacement : bool (default: False) Allows repeated elements if True. Returns comb : int Number of possible combinations. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/math/num_combinations/ num_permutations num_permutations(n, k, with_replacement=False) Function to calculate the number of possible permutations. Parameters n : int Total number of items. k : int Number of elements of the target itemset. with_replacement : bool Allows repeated elements if True. Returns permut : int Number of possible permutations. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/math/num_permutations/ vectorspace_dimensionality vectorspace_dimensionality(ary) Computes the hyper-volume spanned by a vector set Parameters ary : array-like, shape=[num_vectors, num_vectors] An orthogonal set of vectors (arranged as columns in a matrix) Returns dimensions : int An integer indicating the \"dimensionality\" hyper-volume spanned by the vector set Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/math/vectorspace_dimensionality/ vectorspace_orthonormalization vectorspace_orthonormalization(ary, eps=1e-13) Transforms a set of column vectors to a orthonormal basis. Given a set of orthogonal vectors, this functions converts such column vectors, arranged in a matrix, into orthonormal basis vectors. Parameters ary : array-like, shape=[num_vectors, num_vectors] An orthogonal set of vectors (arranged as columns in a matrix) eps : float (default: 1e-13) A small tolerance value to determine whether the vector norm is zero or not. Returns arr : array-like, shape=[num_vectors, num_vectors] An orthonormal set of vectors (arranged as columns) Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/math/vectorspace_orthonormalization/","title":"Mlxtend.math"},{"location":"api_subpackages/mlxtend.math/#factorial","text":"factorial(n) None","title":"factorial"},{"location":"api_subpackages/mlxtend.math/#num_combinations","text":"num_combinations(n, k, with_replacement=False) Function to calculate the number of possible combinations. Parameters n : int Total number of items. k : int Number of elements of the target itemset. with_replacement : bool (default: False) Allows repeated elements if True. Returns comb : int Number of possible combinations. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/math/num_combinations/","title":"num_combinations"},{"location":"api_subpackages/mlxtend.math/#num_permutations","text":"num_permutations(n, k, with_replacement=False) Function to calculate the number of possible permutations. Parameters n : int Total number of items. k : int Number of elements of the target itemset. with_replacement : bool Allows repeated elements if True. Returns permut : int Number of possible permutations. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/math/num_permutations/","title":"num_permutations"},{"location":"api_subpackages/mlxtend.math/#vectorspace_dimensionality","text":"vectorspace_dimensionality(ary) Computes the hyper-volume spanned by a vector set Parameters ary : array-like, shape=[num_vectors, num_vectors] An orthogonal set of vectors (arranged as columns in a matrix) Returns dimensions : int An integer indicating the \"dimensionality\" hyper-volume spanned by the vector set Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/math/vectorspace_dimensionality/","title":"vectorspace_dimensionality"},{"location":"api_subpackages/mlxtend.math/#vectorspace_orthonormalization","text":"vectorspace_orthonormalization(ary, eps=1e-13) Transforms a set of column vectors to a orthonormal basis. Given a set of orthogonal vectors, this functions converts such column vectors, arranged in a matrix, into orthonormal basis vectors. Parameters ary : array-like, shape=[num_vectors, num_vectors] An orthogonal set of vectors (arranged as columns in a matrix) eps : float (default: 1e-13) A small tolerance value to determine whether the vector norm is zero or not. Returns arr : array-like, shape=[num_vectors, num_vectors] An orthonormal set of vectors (arranged as columns) Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/math/vectorspace_orthonormalization/","title":"vectorspace_orthonormalization"},{"location":"api_subpackages/mlxtend.plotting/","text":"mlxtend version: 0.23.1 category_scatter category_scatter(x, y, label_col, data, markers='sxo^v', colors=('blue', 'green', 'red', 'purple', 'gray', 'cyan'), alpha=0.7, markersize=20.0, legend_loc='best') Scatter plot to plot categories in different colors/markerstyles. Parameters x : str or int DataFrame column name of the x-axis values or integer for the numpy ndarray column index. y : str DataFrame column name of the y-axis values or integer for the numpy ndarray column index data : Pandas DataFrame object or NumPy ndarray. markers : str Markers that are cycled through the label category. colors : tuple Colors that are cycled through the label category. alpha : float (default: 0.7) Parameter to control the transparency. markersize : float (default` : 20.0) Parameter to control the marker size. legend_loc : str (default: 'best') Location of the plot legend {best, upper left, upper right, lower left, lower right} No legend if legend_loc=False Returns fig : matplotlig.pyplot figure object Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/plotting/category_scatter/ checkerboard_plot checkerboard_plot(ary, cell_colors=('white', 'black'), font_colors=('black', 'white'), fmt='%.1f', figsize=None, row_labels=None, col_labels=None, fontsize=None) Plot a checkerboard table / heatmap via matplotlib. Parameters ary : array-like, shape = [n, m] A 2D Nnumpy array. cell_colors : tuple or list (default: ('white', 'black')) Tuple or list containing the two colors of the checkerboard pattern. font_colors : tuple or list (default: ('black', 'white')) Font colors corresponding to the cell colors. figsize : tuple (default: (2.5, 2.5)) Height and width of the figure fmt : str (default: '%.1f') Python string formatter for cell values. The default '%.1f' results in floats with 1 digit after the decimal point. Use '%d' to show numbers as integers. row_labels : list (default: None) List of the row labels. Uses the array row indices 0 to n by default. col_labels : list (default: None) List of the column labels. Uses the array column indices 0 to m by default. fontsize : int (default: None) Specifies the font size of the checkerboard table. Uses matplotlib's default if None. Returns fig : matplotlib Figure object. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/plotting/checkerboard_plot/ ecdf ecdf(x, y_label='ECDF', x_label=None, ax=None, percentile=None, ecdf_color=None, ecdf_marker='o', percentile_color='black', percentile_linestyle='--') Plots an Empirical Cumulative Distribution Function Parameters x : array or list, shape=[n_samples,] Array-like object containing the feature values y_label : str (default='ECDF') Text label for the y-axis x_label : str (default=None) Text label for the x-axis ax : matplotlib.axes.Axes (default: None) An existing matplotlib Axes. Creates one if ax=None percentile : float (default=None) Float between 0 and 1 for plotting a percentile threshold line ecdf_color : matplotlib color (default=None) Color for the ECDF plot; uses matplotlib defaults if None ecdf_marker : matplotlib marker (default='o') Marker style for the ECDF plot percentile_color : matplotlib color (default='black') Color for the percentile threshold if percentile is not None percentile_linestyle : matplotlib linestyle (default='--') Line style for the percentile threshold if percentile is not None Returns ax : matplotlib.axes.Axes object percentile_threshold : float Feature threshold at the percentile or None if percentile=None percentile_count : Number of if percentile is not None Number of samples that have a feature less or equal than the feature threshold at a percentile threshold or None if percentile=None Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/plotting/ecdf/ enrichment_plot enrichment_plot(df, colors='bgrkcy', markers=' ', linestyles='-', alpha=0.5, lw=2, where='post', grid=True, count_label='Count', xlim='auto', ylim='auto', invert_axes=False, legend_loc='best', ax=None) Plot stacked barplots Parameters df : pandas.DataFrame A pandas DataFrame where columns represent the different categories. colors: str (default: 'bgrcky') The colors of the bars. markers : str (default: ' ') Matplotlib markerstyles, e.g, 'sov' for square,circle, and triangle markers. linestyles : str (default: '-') Matplotlib linestyles, e.g., '-,--' to cycle normal and dashed lines. Note that the different linestyles need to be separated by commas. alpha : float (default: 0.5) Transparency level from 0.0 to 1.0. lw : int or float (default: 2) Linewidth parameter. where : {'post', 'pre', 'mid'} (default: 'post') Starting location of the steps. grid : bool (default: True ) Plots a grid if True. count_label : str (default: 'Count') Label for the \"Count\"-axis. xlim : 'auto' or array-like [min, max] (default: 'auto') Min and maximum position of the x-axis range. ylim : 'auto' or array-like [min, max] (default: 'auto') Min and maximum position of the y-axis range. invert_axes : bool (default: False) Plots count on the x-axis if True. legend_loc : str (default: 'best') Location of the plot legend {best, upper left, upper right, lower left, lower right} No legend if legend_loc=False ax : matplotlib axis, optional (default: None) Use this axis for plotting or make a new one otherwise Returns ax : matplotlib axis Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/plotting/enrichment_plot/ heatmap heatmap(matrix, hide_spines=False, hide_ticks=False, figsize=None, cmap=None, colorbar=True, row_names=None, column_names=None, column_name_rotation=45, cell_values=True, cell_fmt='.2f', cell_font_size=None, text_color_threshold=None) Plot a heatmap via matplotlib. Parameters conf_mat : array-like, shape = [n_rows, n_columns] And arbitrary 2D array. hide_spines : bool (default: False) Hides axis spines if True. hide_ticks : bool (default: False) Hides axis ticks if True figsize : tuple (default: (2.5, 2.5)) Height and width of the figure cmap : matplotlib colormap (default: None ) Uses matplotlib.pyplot.cm.viridis if None colorbar : bool (default: True) Shows a colorbar if True row_names : array-like, shape = [n_rows] (default: None) List of row names to be used as y-axis tick labels. column_names : array-like, shape = [n_columns] (default: None) List of column names to be used as x-axis tick labels. column_name_rotation : int (default: 45) Number of degrees for rotating column x-tick labels. cell_values : bool (default: True) Plots cell values if True. cell_fmt : string (default: '.2f') Format specification for cell values (if cell_values=True ) cell_font_size : int (default: None) Font size for cell values (if cell_values=True ) text_color_threshold : float (default: None) Threshold for the black/white text threshold of the text annotation. Default (None) tried to infer a good threshold automatically using np.max(normed_matrix) / 2 . Returns fig, ax : matplotlib.pyplot subplot objects Figure and axis elements of the subplot. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/plotting/heatmap/ plot_confusion_matrix plot_confusion_matrix(conf_mat, hide_spines=False, hide_ticks=False, figsize=None, cmap=None, colorbar=False, show_absolute=True, show_normed=False, norm_colormap=None, class_names=None, figure=None, axis=None, fontcolor_threshold=0.5) Plot a confusion matrix via matplotlib. Parameters conf_mat : array-like, shape = [n_classes, n_classes] Confusion matrix from evaluate.confusion matrix. hide_spines : bool (default: False) Hides axis spines if True. hide_ticks : bool (default: False) Hides axis ticks if True figsize : tuple (default: (2.5, 2.5)) Height and width of the figure cmap : matplotlib colormap (default: None ) Uses matplotlib.pyplot.cm.Blues if None colorbar : bool (default: False) Shows a colorbar if True show_absolute : bool (default: True) Shows absolute confusion matrix coefficients if True. At least one of show_absolute or show_normed must be True. show_normed : bool (default: False) Shows normed confusion matrix coefficients if True. The normed confusion matrix coefficients give the proportion of training examples per class that are assigned the correct label. At least one of show_absolute or show_normed must be True. norm_colormap : bool (default: False) Matplotlib color normalization object to normalize the color scale, e.g., matplotlib.colors.LogNorm() . class_names : array-like, shape = [n_classes] (default: None) List of class names. If not None , ticks will be set to these values. figure : None or Matplotlib figure (default: None) If None will create a new figure. axis : None or Matplotlib figure axis (default: None) If None will create a new axis. fontcolor_threshold : Float (default: 0.5) Sets a threshold for choosing black and white font colors for the cells. By default all values larger than 0.5 times the maximum cell value are converted to white, and everything equal or smaller than 0.5 times the maximum cell value are converted to black. Returns fig, ax : matplotlib.pyplot subplot objects Figure and axis elements of the subplot. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/plotting/plot_confusion_matrix/ plot_decision_regions plot_decision_regions(X, y, clf, feature_index=None, filler_feature_values=None, filler_feature_ranges=None, ax=None, X_highlight=None, zoom_factor=1.0, legend=1, hide_spines=True, markers='s^oxv<>', colors='#1f77b4,#ff7f0e,#3ca02c,#d62728,#9467bd,#8c564b,#e377c2,#7f7f7f,#bcbd22,#17becf', scatter_kwargs=None, contourf_kwargs=None, contour_kwargs=None, scatter_highlight_kwargs=None, n_jobs=None) Plot decision regions of a classifier. Please note that this functions assumes that class labels are labeled consecutively, e.g,. 0, 1, 2, 3, 4, and 5. If you have class labels with integer labels > 4, you may want to provide additional colors and/or markers as `colors` and `markers` arguments. See https://matplotlib.org/examples/color/named_colors.html for more information. Parameters X : array-like, shape = [n_samples, n_features] Feature Matrix. y : array-like, shape = [n_samples] True class labels. clf : Classifier object. Must have a .predict method. feature_index : array-like (default: (0,) for 1D, (0, 1) otherwise) Feature indices to use for plotting. The first index in feature_index will be on the x-axis, the second index will be on the y-axis. filler_feature_values : dict (default: None) Only needed for number features > 2. Dictionary of feature index-value pairs for the features not being plotted. filler_feature_ranges : dict (default: None) Only needed for number features > 2. Dictionary of feature index-value pairs for the features not being plotted. Will use the ranges provided to select training samples for plotting. ax : matplotlib.axes.Axes (default: None) An existing matplotlib Axes. Creates one if ax=None. X_highlight : array-like, shape = [n_samples, n_features] (default: None) An array with data points that are used to highlight samples in X . zoom_factor : float (default: 1.0) Controls the scale of the x- and y-axis of the decision plot. hide_spines : bool (default: True) Hide axis spines if True. legend : int (default: 1) Integer to specify the legend location. No legend if legend is 0. markers : str (default: 's^oxv<>') Scatterplot markers. colors : str (default: 'red,blue,limegreen,gray,cyan') Comma separated list of colors. scatter_kwargs : dict (default: None) Keyword arguments for underlying matplotlib scatter function. contourf_kwargs : dict (default: None) Keyword arguments for underlying matplotlib contourf function. contour_kwargs : dict (default: None) Keyword arguments for underlying matplotlib contour function (which draws the lines between decision regions). scatter_highlight_kwargs : dict (default: None) Keyword arguments for underlying matplotlib scatter function. n_jobs : int or None, optional (default=None) The number of CPUs to use to do the computation using Python's multiprocessing library. None means 1. -1 means using all processors. New in v0.22.0. Returns ax : matplotlib.axes.Axes object Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/plotting/plot_decision_regions/ plot_learning_curves plot_learning_curves(X_train, y_train, X_test, y_test, clf, train_marker='o', test_marker='^', scoring='misclassification error', suppress_plot=False, print_model=True, title_fontsize=12, style='default', legend_loc='best') Plots learning curves of a classifier. Parameters X_train : array-like, shape = [n_samples, n_features] Feature matrix of the training dataset. y_train : array-like, shape = [n_samples] True class labels of the training dataset. X_test : array-like, shape = [n_samples, n_features] Feature matrix of the test dataset. y_test : array-like, shape = [n_samples] True class labels of the test dataset. clf : Classifier object. Must have a .predict .fit method. train_marker : str (default: 'o') Marker for the training set line plot. test_marker : str (default: '^') Marker for the test set line plot. scoring : str (default: 'misclassification error') If not 'misclassification error', accepts the following metrics (from scikit-learn): {'accuracy', 'average_precision', 'f1_micro', 'f1_macro', 'f1_weighted', 'f1_samples', 'log_loss', 'precision', 'recall', 'roc_auc', 'adjusted_rand_score', 'mean_absolute_error', 'mean_squared_error', 'median_absolute_error', 'r2'} suppress_plot=False : bool (default: False) Suppress matplotlib plots if True. Recommended for testing purposes. print_model : bool (default: True) Print model parameters in plot title if True. title_fontsize : int (default: 12) Determines the size of the plot title font. style : str (default: 'default') Matplotlib style. For more styles, please see https://matplotlib.org/stable/gallery/style_sheets/style_sheets_reference.html legend_loc : str (default: 'best') Where to place the plot legend: {'best', 'upper left', 'upper right', 'lower left', 'lower right'} Returns errors : (training_error, test_error): tuple of lists Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/plotting/plot_learning_curves/ plot_linear_regression plot_linear_regression(X, y, model=LinearRegression(), corr_func='pearsonr', scattercolor='blue', fit_style='k--', legend=True, xlim='auto') Plot a linear regression line fit. Parameters X : numpy array, shape = [n_samples,] Samples. y : numpy array, shape (n_samples,) Target values model: object (default: sklearn.linear_model.LinearRegression) Estimator object for regression. Must implement a .fit() and .predict() method. corr_func: str or function (default: 'pearsonr') Uses pearsonr from scipy.stats if corr_func='pearsonr'. to compute the regression slope. If not 'pearsonr', the corr_func , the corr_func parameter expects a function of the form func( , ) as inputs, which is expected to return a tuple (<correlation_coefficient>, <some_unused_value>) . scattercolor: string (default: blue) Color of scatter plot points. fit_style: string (default: k--) Style for the line fit. legend: bool (default: True) Plots legend with corr_coeff coef., fit coef., and intercept values. xlim: array-like (x_min, x_max) or 'auto' (default: 'auto') X-axis limits for the linear line fit. Returns regression_fit : tuple intercept, slope, corr_coeff (float, float, float) Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/plotting/plot_linear_regression/ plot_pca_correlation_graph plot_pca_correlation_graph(X, variables_names, dimensions=(1, 2), figure_axis_size=6, X_pca=None, explained_variance=None) Compute the PCA for X and plots the Correlation graph Parameters X : 2d array like. The columns represent the different variables and the rows are the samples of thos variables variables_names : array like Name of the columns (the variables) of X dimensions: tuple with two elements. dimensions to be plotted (x,y) figure_axis_size : size of the final frame. The figure created is a square with length and width equal to figure_axis_size. X_pca : np.ndarray, shape = [n_samples, n_components]. Optional. X_pca is the matrix of the transformed components from X. If not provided, the function computes PCA automatically using mlxtend.feature_extraction.PrincipalComponentAnalysis Expected n_componentes >= max(dimensions) explained_variance : 1 dimension np.ndarray, length = n_components Optional. explained_variance are the eigenvalues from the diagonalized covariance matrix on the PCA transformatiopn. If not provided, the function computes PCA independently Expected n_componentes == X.shape[1] Returns matplotlib_figure, correlation_matrix Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/plotting/plot_pca_correlation_graph/ plot_sequential_feature_selection plot_sequential_feature_selection(metric_dict, figsize=None, kind='std_dev', color='blue', bcolor='steelblue', marker='o', alpha=0.2, ylabel='Performance', confidence_interval=0.95) Plot feature selection results. Parameters metric_dict : mlxtend.SequentialFeatureSelector.get_metric_dict() object figsize : tuple (default: None) Height and width of the figure kind : str (default: \"std_dev\") The kind of error bar or confidence interval in {'std_dev', 'std_err', 'ci', None}. color : str (default: \"blue\") Color of the lineplot (accepts any matplotlib color name) bcolor : str (default: \"steelblue\"). Color of the error bars / confidence intervals (accepts any matplotlib color name). marker : str (default: \"o\") Marker of the line plot (accepts any matplotlib marker name). alpha : float in [0, 1] (default: 0.2) Transparency of the error bars / confidence intervals. ylabel : str (default: \"Performance\") Y-axis label. confidence_interval : float (default: 0.95) Confidence level if kind='ci' . Returns fig : matplotlib.pyplot.figure() object Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/plotting/plot_sequential_feature_selection/ remove_borders remove_borders(axes, left=False, bottom=False, right=True, top=True) Remove chart junk from matplotlib plots. Parameters axes : iterable An iterable containing plt.gca() or plt.subplot() objects, e.g. [plt.gca()]. left : bool (default: False ) Hide left axis spine if True. bottom : bool (default: False ) Hide bottom axis spine if True. right : bool (default: True ) Hide right axis spine if True. top : bool (default: True ) Hide top axis spine if True. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/plotting/remove_chartjunk/ scatter_hist scatter_hist(x, y, xlabel=None, ylabel=None, figsize=(5, 5)) Scatter plot and individual feature histograms along axes. Parameters x : 1D array-like or Pandas Series X-axis values. y : 1D array-like or Pandas Series Y-axis values. xlabel : str (default: None ) Label for the X-axis values. If x is a pandas Series, and xlabel is None , the label is inferred automatically. ylabel : str (default: None ) Label for the X-axis values. If y is a pandas Series, and ylabel is None , the label is inferred automatically. figsize : tuple (default: (5, 5) ) Matplotlib figure size. Returns plot : Matplotlib Figure object scatterplotmatrix scatterplotmatrix(X, fig_axes=None, names=None, figsize=(8, 8), alpha=1.0, kwargs) Lower triangular of a scatterplot matrix Parameters X : array-like, shape={num_examples, num_features} Design matrix containing data instances (examples) with multiple exploratory variables (features). fix_axes : tuple (default: None) A (fig, axes) tuple, where fig is an figure object and axes is an axes object created via matplotlib, for example, by calling the pyplot subplot function fig, axes = plt.subplots(...) names : list (default: None) A list of string names, which should have the same number of elements as there are features (columns) in X . figsize : tuple (default: (8, 8)) Height and width of the subplot grid. Ignored if fig_axes is not None . alpha : float (default: 1.0) Transparency for both the scatter plots and the histograms along the diagonal. **kwargs : kwargs Keyword arguments for the scatterplots. Returns fix_axes : tuple A (fig, axes) tuple, where fig is an figure object and axes is an axes object created via matplotlib, for example, by calling the pyplot subplot function fig, axes = plt.subplots(...) Examples For more usage examples, please see https://rasbt.github.io/mlxtend/user_guide/plotting/scatterplotmatrix/ stacked_barplot stacked_barplot(df, bar_width='auto', colors='bgrcky', labels='index', rotation=90, legend_loc='best') Function to plot stacked barplots Parameters df : pandas.DataFrame A pandas DataFrame where the index denotes the x-axis labels, and the columns contain the different measurements for each row. bar_width: 'auto' or float (default: 'auto') Parameter to set the widths of the bars. if 'auto', the width is automatically determined by the number of columns in the dataset. colors: str (default: 'bgrcky') The colors of the bars. labels: 'index' or iterable (default: 'index') If 'index', the DataFrame index will be used as x-tick labels. rotation: int (default: 90) Parameter to rotate the x-axis labels. legend_loc : str (default: 'best') Location of the plot legend {best, upper left, upper right, lower left, lower right} No legend if legend_loc=False Returns fig : matplotlib.pyplot figure object Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/plotting/stacked_barplot/","title":"Mlxtend.plotting"},{"location":"api_subpackages/mlxtend.plotting/#category_scatter","text":"category_scatter(x, y, label_col, data, markers='sxo^v', colors=('blue', 'green', 'red', 'purple', 'gray', 'cyan'), alpha=0.7, markersize=20.0, legend_loc='best') Scatter plot to plot categories in different colors/markerstyles. Parameters x : str or int DataFrame column name of the x-axis values or integer for the numpy ndarray column index. y : str DataFrame column name of the y-axis values or integer for the numpy ndarray column index data : Pandas DataFrame object or NumPy ndarray. markers : str Markers that are cycled through the label category. colors : tuple Colors that are cycled through the label category. alpha : float (default: 0.7) Parameter to control the transparency. markersize : float (default` : 20.0) Parameter to control the marker size. legend_loc : str (default: 'best') Location of the plot legend {best, upper left, upper right, lower left, lower right} No legend if legend_loc=False Returns fig : matplotlig.pyplot figure object Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/plotting/category_scatter/","title":"category_scatter"},{"location":"api_subpackages/mlxtend.plotting/#checkerboard_plot","text":"checkerboard_plot(ary, cell_colors=('white', 'black'), font_colors=('black', 'white'), fmt='%.1f', figsize=None, row_labels=None, col_labels=None, fontsize=None) Plot a checkerboard table / heatmap via matplotlib. Parameters ary : array-like, shape = [n, m] A 2D Nnumpy array. cell_colors : tuple or list (default: ('white', 'black')) Tuple or list containing the two colors of the checkerboard pattern. font_colors : tuple or list (default: ('black', 'white')) Font colors corresponding to the cell colors. figsize : tuple (default: (2.5, 2.5)) Height and width of the figure fmt : str (default: '%.1f') Python string formatter for cell values. The default '%.1f' results in floats with 1 digit after the decimal point. Use '%d' to show numbers as integers. row_labels : list (default: None) List of the row labels. Uses the array row indices 0 to n by default. col_labels : list (default: None) List of the column labels. Uses the array column indices 0 to m by default. fontsize : int (default: None) Specifies the font size of the checkerboard table. Uses matplotlib's default if None. Returns fig : matplotlib Figure object. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/plotting/checkerboard_plot/","title":"checkerboard_plot"},{"location":"api_subpackages/mlxtend.plotting/#ecdf","text":"ecdf(x, y_label='ECDF', x_label=None, ax=None, percentile=None, ecdf_color=None, ecdf_marker='o', percentile_color='black', percentile_linestyle='--') Plots an Empirical Cumulative Distribution Function Parameters x : array or list, shape=[n_samples,] Array-like object containing the feature values y_label : str (default='ECDF') Text label for the y-axis x_label : str (default=None) Text label for the x-axis ax : matplotlib.axes.Axes (default: None) An existing matplotlib Axes. Creates one if ax=None percentile : float (default=None) Float between 0 and 1 for plotting a percentile threshold line ecdf_color : matplotlib color (default=None) Color for the ECDF plot; uses matplotlib defaults if None ecdf_marker : matplotlib marker (default='o') Marker style for the ECDF plot percentile_color : matplotlib color (default='black') Color for the percentile threshold if percentile is not None percentile_linestyle : matplotlib linestyle (default='--') Line style for the percentile threshold if percentile is not None Returns ax : matplotlib.axes.Axes object percentile_threshold : float Feature threshold at the percentile or None if percentile=None percentile_count : Number of if percentile is not None Number of samples that have a feature less or equal than the feature threshold at a percentile threshold or None if percentile=None Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/plotting/ecdf/","title":"ecdf"},{"location":"api_subpackages/mlxtend.plotting/#enrichment_plot","text":"enrichment_plot(df, colors='bgrkcy', markers=' ', linestyles='-', alpha=0.5, lw=2, where='post', grid=True, count_label='Count', xlim='auto', ylim='auto', invert_axes=False, legend_loc='best', ax=None) Plot stacked barplots Parameters df : pandas.DataFrame A pandas DataFrame where columns represent the different categories. colors: str (default: 'bgrcky') The colors of the bars. markers : str (default: ' ') Matplotlib markerstyles, e.g, 'sov' for square,circle, and triangle markers. linestyles : str (default: '-') Matplotlib linestyles, e.g., '-,--' to cycle normal and dashed lines. Note that the different linestyles need to be separated by commas. alpha : float (default: 0.5) Transparency level from 0.0 to 1.0. lw : int or float (default: 2) Linewidth parameter. where : {'post', 'pre', 'mid'} (default: 'post') Starting location of the steps. grid : bool (default: True ) Plots a grid if True. count_label : str (default: 'Count') Label for the \"Count\"-axis. xlim : 'auto' or array-like [min, max] (default: 'auto') Min and maximum position of the x-axis range. ylim : 'auto' or array-like [min, max] (default: 'auto') Min and maximum position of the y-axis range. invert_axes : bool (default: False) Plots count on the x-axis if True. legend_loc : str (default: 'best') Location of the plot legend {best, upper left, upper right, lower left, lower right} No legend if legend_loc=False ax : matplotlib axis, optional (default: None) Use this axis for plotting or make a new one otherwise Returns ax : matplotlib axis Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/plotting/enrichment_plot/","title":"enrichment_plot"},{"location":"api_subpackages/mlxtend.plotting/#heatmap","text":"heatmap(matrix, hide_spines=False, hide_ticks=False, figsize=None, cmap=None, colorbar=True, row_names=None, column_names=None, column_name_rotation=45, cell_values=True, cell_fmt='.2f', cell_font_size=None, text_color_threshold=None) Plot a heatmap via matplotlib. Parameters conf_mat : array-like, shape = [n_rows, n_columns] And arbitrary 2D array. hide_spines : bool (default: False) Hides axis spines if True. hide_ticks : bool (default: False) Hides axis ticks if True figsize : tuple (default: (2.5, 2.5)) Height and width of the figure cmap : matplotlib colormap (default: None ) Uses matplotlib.pyplot.cm.viridis if None colorbar : bool (default: True) Shows a colorbar if True row_names : array-like, shape = [n_rows] (default: None) List of row names to be used as y-axis tick labels. column_names : array-like, shape = [n_columns] (default: None) List of column names to be used as x-axis tick labels. column_name_rotation : int (default: 45) Number of degrees for rotating column x-tick labels. cell_values : bool (default: True) Plots cell values if True. cell_fmt : string (default: '.2f') Format specification for cell values (if cell_values=True ) cell_font_size : int (default: None) Font size for cell values (if cell_values=True ) text_color_threshold : float (default: None) Threshold for the black/white text threshold of the text annotation. Default (None) tried to infer a good threshold automatically using np.max(normed_matrix) / 2 . Returns fig, ax : matplotlib.pyplot subplot objects Figure and axis elements of the subplot. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/plotting/heatmap/","title":"heatmap"},{"location":"api_subpackages/mlxtend.plotting/#plot_confusion_matrix","text":"plot_confusion_matrix(conf_mat, hide_spines=False, hide_ticks=False, figsize=None, cmap=None, colorbar=False, show_absolute=True, show_normed=False, norm_colormap=None, class_names=None, figure=None, axis=None, fontcolor_threshold=0.5) Plot a confusion matrix via matplotlib. Parameters conf_mat : array-like, shape = [n_classes, n_classes] Confusion matrix from evaluate.confusion matrix. hide_spines : bool (default: False) Hides axis spines if True. hide_ticks : bool (default: False) Hides axis ticks if True figsize : tuple (default: (2.5, 2.5)) Height and width of the figure cmap : matplotlib colormap (default: None ) Uses matplotlib.pyplot.cm.Blues if None colorbar : bool (default: False) Shows a colorbar if True show_absolute : bool (default: True) Shows absolute confusion matrix coefficients if True. At least one of show_absolute or show_normed must be True. show_normed : bool (default: False) Shows normed confusion matrix coefficients if True. The normed confusion matrix coefficients give the proportion of training examples per class that are assigned the correct label. At least one of show_absolute or show_normed must be True. norm_colormap : bool (default: False) Matplotlib color normalization object to normalize the color scale, e.g., matplotlib.colors.LogNorm() . class_names : array-like, shape = [n_classes] (default: None) List of class names. If not None , ticks will be set to these values. figure : None or Matplotlib figure (default: None) If None will create a new figure. axis : None or Matplotlib figure axis (default: None) If None will create a new axis. fontcolor_threshold : Float (default: 0.5) Sets a threshold for choosing black and white font colors for the cells. By default all values larger than 0.5 times the maximum cell value are converted to white, and everything equal or smaller than 0.5 times the maximum cell value are converted to black. Returns fig, ax : matplotlib.pyplot subplot objects Figure and axis elements of the subplot. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/plotting/plot_confusion_matrix/","title":"plot_confusion_matrix"},{"location":"api_subpackages/mlxtend.plotting/#plot_decision_regions","text":"plot_decision_regions(X, y, clf, feature_index=None, filler_feature_values=None, filler_feature_ranges=None, ax=None, X_highlight=None, zoom_factor=1.0, legend=1, hide_spines=True, markers='s^oxv<>', colors='#1f77b4,#ff7f0e,#3ca02c,#d62728,#9467bd,#8c564b,#e377c2,#7f7f7f,#bcbd22,#17becf', scatter_kwargs=None, contourf_kwargs=None, contour_kwargs=None, scatter_highlight_kwargs=None, n_jobs=None) Plot decision regions of a classifier. Please note that this functions assumes that class labels are labeled consecutively, e.g,. 0, 1, 2, 3, 4, and 5. If you have class labels with integer labels > 4, you may want to provide additional colors and/or markers as `colors` and `markers` arguments. See https://matplotlib.org/examples/color/named_colors.html for more information. Parameters X : array-like, shape = [n_samples, n_features] Feature Matrix. y : array-like, shape = [n_samples] True class labels. clf : Classifier object. Must have a .predict method. feature_index : array-like (default: (0,) for 1D, (0, 1) otherwise) Feature indices to use for plotting. The first index in feature_index will be on the x-axis, the second index will be on the y-axis. filler_feature_values : dict (default: None) Only needed for number features > 2. Dictionary of feature index-value pairs for the features not being plotted. filler_feature_ranges : dict (default: None) Only needed for number features > 2. Dictionary of feature index-value pairs for the features not being plotted. Will use the ranges provided to select training samples for plotting. ax : matplotlib.axes.Axes (default: None) An existing matplotlib Axes. Creates one if ax=None. X_highlight : array-like, shape = [n_samples, n_features] (default: None) An array with data points that are used to highlight samples in X . zoom_factor : float (default: 1.0) Controls the scale of the x- and y-axis of the decision plot. hide_spines : bool (default: True) Hide axis spines if True. legend : int (default: 1) Integer to specify the legend location. No legend if legend is 0. markers : str (default: 's^oxv<>') Scatterplot markers. colors : str (default: 'red,blue,limegreen,gray,cyan') Comma separated list of colors. scatter_kwargs : dict (default: None) Keyword arguments for underlying matplotlib scatter function. contourf_kwargs : dict (default: None) Keyword arguments for underlying matplotlib contourf function. contour_kwargs : dict (default: None) Keyword arguments for underlying matplotlib contour function (which draws the lines between decision regions). scatter_highlight_kwargs : dict (default: None) Keyword arguments for underlying matplotlib scatter function. n_jobs : int or None, optional (default=None) The number of CPUs to use to do the computation using Python's multiprocessing library. None means 1. -1 means using all processors. New in v0.22.0. Returns ax : matplotlib.axes.Axes object Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/plotting/plot_decision_regions/","title":"plot_decision_regions"},{"location":"api_subpackages/mlxtend.plotting/#plot_learning_curves","text":"plot_learning_curves(X_train, y_train, X_test, y_test, clf, train_marker='o', test_marker='^', scoring='misclassification error', suppress_plot=False, print_model=True, title_fontsize=12, style='default', legend_loc='best') Plots learning curves of a classifier. Parameters X_train : array-like, shape = [n_samples, n_features] Feature matrix of the training dataset. y_train : array-like, shape = [n_samples] True class labels of the training dataset. X_test : array-like, shape = [n_samples, n_features] Feature matrix of the test dataset. y_test : array-like, shape = [n_samples] True class labels of the test dataset. clf : Classifier object. Must have a .predict .fit method. train_marker : str (default: 'o') Marker for the training set line plot. test_marker : str (default: '^') Marker for the test set line plot. scoring : str (default: 'misclassification error') If not 'misclassification error', accepts the following metrics (from scikit-learn): {'accuracy', 'average_precision', 'f1_micro', 'f1_macro', 'f1_weighted', 'f1_samples', 'log_loss', 'precision', 'recall', 'roc_auc', 'adjusted_rand_score', 'mean_absolute_error', 'mean_squared_error', 'median_absolute_error', 'r2'} suppress_plot=False : bool (default: False) Suppress matplotlib plots if True. Recommended for testing purposes. print_model : bool (default: True) Print model parameters in plot title if True. title_fontsize : int (default: 12) Determines the size of the plot title font. style : str (default: 'default') Matplotlib style. For more styles, please see https://matplotlib.org/stable/gallery/style_sheets/style_sheets_reference.html legend_loc : str (default: 'best') Where to place the plot legend: {'best', 'upper left', 'upper right', 'lower left', 'lower right'} Returns errors : (training_error, test_error): tuple of lists Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/plotting/plot_learning_curves/","title":"plot_learning_curves"},{"location":"api_subpackages/mlxtend.plotting/#plot_linear_regression","text":"plot_linear_regression(X, y, model=LinearRegression(), corr_func='pearsonr', scattercolor='blue', fit_style='k--', legend=True, xlim='auto') Plot a linear regression line fit. Parameters X : numpy array, shape = [n_samples,] Samples. y : numpy array, shape (n_samples,) Target values model: object (default: sklearn.linear_model.LinearRegression) Estimator object for regression. Must implement a .fit() and .predict() method. corr_func: str or function (default: 'pearsonr') Uses pearsonr from scipy.stats if corr_func='pearsonr'. to compute the regression slope. If not 'pearsonr', the corr_func , the corr_func parameter expects a function of the form func( , ) as inputs, which is expected to return a tuple (<correlation_coefficient>, <some_unused_value>) . scattercolor: string (default: blue) Color of scatter plot points. fit_style: string (default: k--) Style for the line fit. legend: bool (default: True) Plots legend with corr_coeff coef., fit coef., and intercept values. xlim: array-like (x_min, x_max) or 'auto' (default: 'auto') X-axis limits for the linear line fit. Returns regression_fit : tuple intercept, slope, corr_coeff (float, float, float) Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/plotting/plot_linear_regression/","title":"plot_linear_regression"},{"location":"api_subpackages/mlxtend.plotting/#plot_pca_correlation_graph","text":"plot_pca_correlation_graph(X, variables_names, dimensions=(1, 2), figure_axis_size=6, X_pca=None, explained_variance=None) Compute the PCA for X and plots the Correlation graph Parameters X : 2d array like. The columns represent the different variables and the rows are the samples of thos variables variables_names : array like Name of the columns (the variables) of X dimensions: tuple with two elements. dimensions to be plotted (x,y) figure_axis_size : size of the final frame. The figure created is a square with length and width equal to figure_axis_size. X_pca : np.ndarray, shape = [n_samples, n_components]. Optional. X_pca is the matrix of the transformed components from X. If not provided, the function computes PCA automatically using mlxtend.feature_extraction.PrincipalComponentAnalysis Expected n_componentes >= max(dimensions) explained_variance : 1 dimension np.ndarray, length = n_components Optional. explained_variance are the eigenvalues from the diagonalized covariance matrix on the PCA transformatiopn. If not provided, the function computes PCA independently Expected n_componentes == X.shape[1] Returns matplotlib_figure, correlation_matrix Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/plotting/plot_pca_correlation_graph/","title":"plot_pca_correlation_graph"},{"location":"api_subpackages/mlxtend.plotting/#plot_sequential_feature_selection","text":"plot_sequential_feature_selection(metric_dict, figsize=None, kind='std_dev', color='blue', bcolor='steelblue', marker='o', alpha=0.2, ylabel='Performance', confidence_interval=0.95) Plot feature selection results. Parameters metric_dict : mlxtend.SequentialFeatureSelector.get_metric_dict() object figsize : tuple (default: None) Height and width of the figure kind : str (default: \"std_dev\") The kind of error bar or confidence interval in {'std_dev', 'std_err', 'ci', None}. color : str (default: \"blue\") Color of the lineplot (accepts any matplotlib color name) bcolor : str (default: \"steelblue\"). Color of the error bars / confidence intervals (accepts any matplotlib color name). marker : str (default: \"o\") Marker of the line plot (accepts any matplotlib marker name). alpha : float in [0, 1] (default: 0.2) Transparency of the error bars / confidence intervals. ylabel : str (default: \"Performance\") Y-axis label. confidence_interval : float (default: 0.95) Confidence level if kind='ci' . Returns fig : matplotlib.pyplot.figure() object Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/plotting/plot_sequential_feature_selection/","title":"plot_sequential_feature_selection"},{"location":"api_subpackages/mlxtend.plotting/#remove_borders","text":"remove_borders(axes, left=False, bottom=False, right=True, top=True) Remove chart junk from matplotlib plots. Parameters axes : iterable An iterable containing plt.gca() or plt.subplot() objects, e.g. [plt.gca()]. left : bool (default: False ) Hide left axis spine if True. bottom : bool (default: False ) Hide bottom axis spine if True. right : bool (default: True ) Hide right axis spine if True. top : bool (default: True ) Hide top axis spine if True. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/plotting/remove_chartjunk/","title":"remove_borders"},{"location":"api_subpackages/mlxtend.plotting/#scatter_hist","text":"scatter_hist(x, y, xlabel=None, ylabel=None, figsize=(5, 5)) Scatter plot and individual feature histograms along axes. Parameters x : 1D array-like or Pandas Series X-axis values. y : 1D array-like or Pandas Series Y-axis values. xlabel : str (default: None ) Label for the X-axis values. If x is a pandas Series, and xlabel is None , the label is inferred automatically. ylabel : str (default: None ) Label for the X-axis values. If y is a pandas Series, and ylabel is None , the label is inferred automatically. figsize : tuple (default: (5, 5) ) Matplotlib figure size. Returns plot : Matplotlib Figure object","title":"scatter_hist"},{"location":"api_subpackages/mlxtend.plotting/#scatterplotmatrix","text":"scatterplotmatrix(X, fig_axes=None, names=None, figsize=(8, 8), alpha=1.0, kwargs) Lower triangular of a scatterplot matrix Parameters X : array-like, shape={num_examples, num_features} Design matrix containing data instances (examples) with multiple exploratory variables (features). fix_axes : tuple (default: None) A (fig, axes) tuple, where fig is an figure object and axes is an axes object created via matplotlib, for example, by calling the pyplot subplot function fig, axes = plt.subplots(...) names : list (default: None) A list of string names, which should have the same number of elements as there are features (columns) in X . figsize : tuple (default: (8, 8)) Height and width of the subplot grid. Ignored if fig_axes is not None . alpha : float (default: 1.0) Transparency for both the scatter plots and the histograms along the diagonal. **kwargs : kwargs Keyword arguments for the scatterplots. Returns fix_axes : tuple A (fig, axes) tuple, where fig is an figure object and axes is an axes object created via matplotlib, for example, by calling the pyplot subplot function fig, axes = plt.subplots(...) Examples For more usage examples, please see https://rasbt.github.io/mlxtend/user_guide/plotting/scatterplotmatrix/","title":"scatterplotmatrix"},{"location":"api_subpackages/mlxtend.plotting/#stacked_barplot","text":"stacked_barplot(df, bar_width='auto', colors='bgrcky', labels='index', rotation=90, legend_loc='best') Function to plot stacked barplots Parameters df : pandas.DataFrame A pandas DataFrame where the index denotes the x-axis labels, and the columns contain the different measurements for each row. bar_width: 'auto' or float (default: 'auto') Parameter to set the widths of the bars. if 'auto', the width is automatically determined by the number of columns in the dataset. colors: str (default: 'bgrcky') The colors of the bars. labels: 'index' or iterable (default: 'index') If 'index', the DataFrame index will be used as x-tick labels. rotation: int (default: 90) Parameter to rotate the x-axis labels. legend_loc : str (default: 'best') Location of the plot legend {best, upper left, upper right, lower left, lower right} No legend if legend_loc=False Returns fig : matplotlib.pyplot figure object Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/plotting/stacked_barplot/","title":"stacked_barplot"},{"location":"api_subpackages/mlxtend.preprocessing/","text":"mlxtend version: 0.23.1 CopyTransformer CopyTransformer() Transformer that returns a copy of the input array For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/preprocessing/CopyTransformer/ Methods fit(X, y=None) Mock method. Does nothing. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] (default: None) Returns self fit_transform(X, y=None) Return a copy of the input array. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] (default: None) Returns X_copy : copy of the input X array. get_params(deep=True) Get parameters for this estimator. Parameters deep : bool, default=True If True, will return the parameters for this estimator and contained subobjects that are estimators. Returns params : dict Parameter names mapped to their values. set_params( params) Set the parameters of this estimator. The method works on simple estimators as well as on nested objects (such as :class:`~sklearn.pipeline.Pipeline`). The latter have parameters of the form ``<component>__<parameter>`` so that it's possible to update each component of a nested object. Parameters **params : dict Estimator parameters. Returns self : estimator instance Estimator instance. transform(X, y=None) Return a copy of the input array. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] (default: None) Returns X_copy : copy of the input X array. DenseTransformer DenseTransformer(return_copy=True) Convert a sparse array into a dense array. For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/preprocessing/DenseTransformer/ Methods fit(X, y=None) Mock method. Does nothing. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] (default: None) Returns self fit_transform(X, y=None) Return a dense version of the input array. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] (default: None) Returns X_dense : dense version of the input X array. get_params(deep=True) Get parameters for this estimator. Parameters deep : bool, default=True If True, will return the parameters for this estimator and contained subobjects that are estimators. Returns params : dict Parameter names mapped to their values. set_params( params) Set the parameters of this estimator. The method works on simple estimators as well as on nested objects (such as :class:`~sklearn.pipeline.Pipeline`). The latter have parameters of the form ``<component>__<parameter>`` so that it's possible to update each component of a nested object. Parameters **params : dict Estimator parameters. Returns self : estimator instance Estimator instance. transform(X, y=None) Return a dense version of the input array. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] (default: None) Returns X_dense : dense version of the input X array. MeanCenterer MeanCenterer() Column centering of vectors and matrices. Attributes col_means : numpy.ndarray [n_columns] NumPy array storing the mean values for centering after fitting the MeanCenterer object. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/preprocessing/MeanCenterer/ Methods fit(X) Gets the column means for mean centering. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Array of data vectors, where n_samples is the number of samples and n_features is the number of features. Returns self fit_transform(X) Fits and transforms an arry. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Array of data vectors, where n_samples is the number of samples and n_features is the number of features. Returns X_tr : {array-like, sparse matrix}, shape = [n_samples, n_features] A copy of the input array with the columns centered. transform(X) Centers a NumPy array. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Array of data vectors, where n_samples is the number of samples and n_features is the number of features. Returns X_tr : {array-like, sparse matrix}, shape = [n_samples, n_features] A copy of the input array with the columns centered. TransactionEncoder TransactionEncoder() Encoder class for transaction data in Python lists Parameters None Attributes columns_: list List of unique names in the X input list of lists Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/preprocessing/TransactionEncoder/ Methods fit(X) Learn unique column names from transaction DataFrame Parameters X : list of lists A python list of lists, where the outer list stores the n transactions and the inner list stores the items in each transaction. For example, [['Apple', 'Beer', 'Rice', 'Chicken'], ['Apple', 'Beer', 'Rice'], ['Apple', 'Beer'], ['Apple', 'Bananas'], ['Milk', 'Beer', 'Rice', 'Chicken'], ['Milk', 'Beer', 'Rice'], ['Milk', 'Beer'], ['Apple', 'Bananas']] fit_transform(X, sparse=False) Fit a TransactionEncoder encoder and transform a dataset. get_params(deep=True) Get parameters for this estimator. Parameters deep : bool, default=True If True, will return the parameters for this estimator and contained subobjects that are estimators. Returns params : dict Parameter names mapped to their values. inverse_transform(array) Transforms an encoded NumPy array back into transactions. Parameters array : NumPy array [n_transactions, n_unique_items] The NumPy one-hot encoded boolean array of the input transactions, where the columns represent the unique items found in the input array in alphabetic order For example, array([[True , False, True , True , False, True ], [True , False, True , False, False, True ], [True , False, True , False, False, False], [True , True , False, False, False, False], [False, False, True , True , True , True ], [False, False, True , False, True , True ], [False, False, True , False, True , False], [True , True , False, False, False, False]]) The corresponding column labels are available as self.columns_, e.g., ['Apple', 'Bananas', 'Beer', 'Chicken', 'Milk', 'Rice'] Returns X : list of lists A python list of lists, where the outer list stores the n transactions and the inner list stores the items in each transaction. For example, [['Apple', 'Beer', 'Rice', 'Chicken'], ['Apple', 'Beer', 'Rice'], ['Apple', 'Beer'], ['Apple', 'Bananas'], ['Milk', 'Beer', 'Rice', 'Chicken'], ['Milk', 'Beer', 'Rice'], ['Milk', 'Beer'], ['Apple', 'Bananas']] set_output( , transform=None)* Set output container. See :ref:`sphx_glr_auto_examples_miscellaneous_plot_set_output.py` for an example on how to use the API. Parameters transform : {\"default\", \"pandas\"}, default=None Configure output of transform and fit_transform . \"default\" : Default output format of a transformer \"pandas\" : DataFrame output None : Transform configuration is unchanged Returns self : estimator instance Estimator instance. set_params( params) Set the parameters of this estimator. The method works on simple estimators as well as on nested objects (such as :class:`~sklearn.pipeline.Pipeline`). The latter have parameters of the form ``<component>__<parameter>`` so that it's possible to update each component of a nested object. Parameters **params : dict Estimator parameters. Returns self : estimator instance Estimator instance. transform(X, sparse=False) Transform transactions into a one-hot encoded NumPy array. Parameters X : list of lists A python list of lists, where the outer list stores the n transactions and the inner list stores the items in each transaction. For example, [['Apple', 'Beer', 'Rice', 'Chicken'], ['Apple', 'Beer', 'Rice'], ['Apple', 'Beer'], ['Apple', 'Bananas'], ['Milk', 'Beer', 'Rice', 'Chicken'], ['Milk', 'Beer', 'Rice'], ['Milk', 'Beer'], ['Apple', 'Bananas']] sparse: bool (default=False) If True, transform will return Compressed Sparse Row matrix instead of the regular one. Returns array : NumPy array [n_transactions, n_unique_items] if sparse=False (default). Compressed Sparse Row matrix otherwise The one-hot encoded boolean array of the input transactions, where the columns represent the unique items found in the input array in alphabetic order. Exact representation depends on the sparse argument For example, array([[True , False, True , True , False, True ], [True , False, True , False, False, True ], [True , False, True , False, False, False], [True , True , False, False, False, False], [False, False, True , True , True , True ], [False, False, True , False, True , True ], [False, False, True , False, True , False], [True , True , False, False, False, False]]) The corresponding column labels are available as self.columns_, e.g., ['Apple', 'Bananas', 'Beer', 'Chicken', 'Milk', 'Rice'] minmax_scaling minmax_scaling(array, columns, min_val=0, max_val=1) Min max scaling of pandas' DataFrames. Parameters array : pandas DataFrame or NumPy ndarray, shape = [n_rows, n_columns]. columns : array-like, shape = [n_columns] Array-like with column names, e.g., ['col1', 'col2', ...] or column indices [0, 2, 4, ...] min_val : int or float , optional (default= 0 ) minimum value after rescaling. max_val : int or float , optional (default= 1 ) maximum value after rescaling. Returns df_new : pandas DataFrame object. Copy of the array or DataFrame with rescaled columns. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/preprocessing/minmax_scaling/ one_hot one_hot(y, num_labels='auto', dtype='float') One-hot encoding of class labels Parameters y : array-like, shape = [n_classlabels] Python list or numpy array consisting of class labels. num_labels : int or 'auto' Number of unique labels in the class label array. Infers the number of unique labels from the input array if set to 'auto'. dtype : str NumPy array type (float, float32, float64) of the output array. Returns ary : numpy.ndarray, shape = [n_classlabels] One-hot encoded array, where each sample is represented as a row vector in the returned array. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/preprocessing/one_hot/ shuffle_arrays_unison shuffle_arrays_unison(arrays, random_seed=None) Shuffle NumPy arrays in unison. Parameters arrays : array-like, shape = [n_arrays] A list of NumPy arrays. random_seed : int (default: None) Sets the random state. Returns shuffled_arrays : A list of NumPy arrays after shuffling. Examples ``` >>> import numpy as np >>> from mlxtend.preprocessing import shuffle_arrays_unison >>> X1 = np.array([[1, 2, 3], [4, 5, 6], [7, 8, 9]]) >>> y1 = np.array([1, 2, 3]) >>> X2, y2 = shuffle_arrays_unison(arrays=[X1, y1], random_seed=3) >>> assert(X2.all() == np.array([[4, 5, 6], [1, 2, 3], [7, 8, 9]]).all()) >>> assert(y2.all() == np.array([2, 1, 3]).all()) >>> For more usage examples, please see https://rasbt.github.io/mlxtend/user_guide/preprocessing/shuffle_arrays_unison/ ``` standardize standardize(array, columns=None, ddof=0, return_params=False, params=None) Standardize columns in pandas DataFrames. Parameters array : pandas DataFrame or NumPy ndarray, shape = [n_rows, n_columns]. columns : array-like, shape = [n_columns] (default: None) Array-like with column names, e.g., ['col1', 'col2', ...] or column indices [0, 2, 4, ...] If None, standardizes all columns. ddof : int (default: 0) Delta Degrees of Freedom. The divisor used in calculations is N - ddof, where N represents the number of elements. return_params : dict (default: False) If set to True, a dictionary is returned in addition to the standardized array. The parameter dictionary contains the column means ('avgs') and standard deviations ('stds') of the individual columns. params : dict (default: None) A dictionary with column means and standard deviations as returned by the standardize function if return_params was set to True. If a params dictionary is provided, the standardize function will use these instead of computing them from the current array. Notes If all values in a given column are the same, these values are all set to 0.0 . The standard deviation in the parameters dictionary is consequently set to 1.0 to avoid dividing by zero. Returns df_new : pandas DataFrame object. Copy of the array or DataFrame with standardized columns. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/preprocessing/standardize/","title":"Mlxtend.preprocessing"},{"location":"api_subpackages/mlxtend.preprocessing/#copytransformer","text":"CopyTransformer() Transformer that returns a copy of the input array For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/preprocessing/CopyTransformer/","title":"CopyTransformer"},{"location":"api_subpackages/mlxtend.preprocessing/#methods","text":"fit(X, y=None) Mock method. Does nothing. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] (default: None) Returns self fit_transform(X, y=None) Return a copy of the input array. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] (default: None) Returns X_copy : copy of the input X array. get_params(deep=True) Get parameters for this estimator. Parameters deep : bool, default=True If True, will return the parameters for this estimator and contained subobjects that are estimators. Returns params : dict Parameter names mapped to their values. set_params( params) Set the parameters of this estimator. The method works on simple estimators as well as on nested objects (such as :class:`~sklearn.pipeline.Pipeline`). The latter have parameters of the form ``<component>__<parameter>`` so that it's possible to update each component of a nested object. Parameters **params : dict Estimator parameters. Returns self : estimator instance Estimator instance. transform(X, y=None) Return a copy of the input array. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] (default: None) Returns X_copy : copy of the input X array.","title":"Methods"},{"location":"api_subpackages/mlxtend.preprocessing/#densetransformer","text":"DenseTransformer(return_copy=True) Convert a sparse array into a dense array. For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/preprocessing/DenseTransformer/","title":"DenseTransformer"},{"location":"api_subpackages/mlxtend.preprocessing/#methods_1","text":"fit(X, y=None) Mock method. Does nothing. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] (default: None) Returns self fit_transform(X, y=None) Return a dense version of the input array. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] (default: None) Returns X_dense : dense version of the input X array. get_params(deep=True) Get parameters for this estimator. Parameters deep : bool, default=True If True, will return the parameters for this estimator and contained subobjects that are estimators. Returns params : dict Parameter names mapped to their values. set_params( params) Set the parameters of this estimator. The method works on simple estimators as well as on nested objects (such as :class:`~sklearn.pipeline.Pipeline`). The latter have parameters of the form ``<component>__<parameter>`` so that it's possible to update each component of a nested object. Parameters **params : dict Estimator parameters. Returns self : estimator instance Estimator instance. transform(X, y=None) Return a dense version of the input array. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] (default: None) Returns X_dense : dense version of the input X array.","title":"Methods"},{"location":"api_subpackages/mlxtend.preprocessing/#meancenterer","text":"MeanCenterer() Column centering of vectors and matrices. Attributes col_means : numpy.ndarray [n_columns] NumPy array storing the mean values for centering after fitting the MeanCenterer object. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/preprocessing/MeanCenterer/","title":"MeanCenterer"},{"location":"api_subpackages/mlxtend.preprocessing/#methods_2","text":"fit(X) Gets the column means for mean centering. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Array of data vectors, where n_samples is the number of samples and n_features is the number of features. Returns self fit_transform(X) Fits and transforms an arry. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Array of data vectors, where n_samples is the number of samples and n_features is the number of features. Returns X_tr : {array-like, sparse matrix}, shape = [n_samples, n_features] A copy of the input array with the columns centered. transform(X) Centers a NumPy array. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Array of data vectors, where n_samples is the number of samples and n_features is the number of features. Returns X_tr : {array-like, sparse matrix}, shape = [n_samples, n_features] A copy of the input array with the columns centered.","title":"Methods"},{"location":"api_subpackages/mlxtend.preprocessing/#transactionencoder","text":"TransactionEncoder() Encoder class for transaction data in Python lists Parameters None Attributes columns_: list List of unique names in the X input list of lists Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/preprocessing/TransactionEncoder/","title":"TransactionEncoder"},{"location":"api_subpackages/mlxtend.preprocessing/#methods_3","text":"fit(X) Learn unique column names from transaction DataFrame Parameters X : list of lists A python list of lists, where the outer list stores the n transactions and the inner list stores the items in each transaction. For example, [['Apple', 'Beer', 'Rice', 'Chicken'], ['Apple', 'Beer', 'Rice'], ['Apple', 'Beer'], ['Apple', 'Bananas'], ['Milk', 'Beer', 'Rice', 'Chicken'], ['Milk', 'Beer', 'Rice'], ['Milk', 'Beer'], ['Apple', 'Bananas']] fit_transform(X, sparse=False) Fit a TransactionEncoder encoder and transform a dataset. get_params(deep=True) Get parameters for this estimator. Parameters deep : bool, default=True If True, will return the parameters for this estimator and contained subobjects that are estimators. Returns params : dict Parameter names mapped to their values. inverse_transform(array) Transforms an encoded NumPy array back into transactions. Parameters array : NumPy array [n_transactions, n_unique_items] The NumPy one-hot encoded boolean array of the input transactions, where the columns represent the unique items found in the input array in alphabetic order For example, array([[True , False, True , True , False, True ], [True , False, True , False, False, True ], [True , False, True , False, False, False], [True , True , False, False, False, False], [False, False, True , True , True , True ], [False, False, True , False, True , True ], [False, False, True , False, True , False], [True , True , False, False, False, False]]) The corresponding column labels are available as self.columns_, e.g., ['Apple', 'Bananas', 'Beer', 'Chicken', 'Milk', 'Rice'] Returns X : list of lists A python list of lists, where the outer list stores the n transactions and the inner list stores the items in each transaction. For example, [['Apple', 'Beer', 'Rice', 'Chicken'], ['Apple', 'Beer', 'Rice'], ['Apple', 'Beer'], ['Apple', 'Bananas'], ['Milk', 'Beer', 'Rice', 'Chicken'], ['Milk', 'Beer', 'Rice'], ['Milk', 'Beer'], ['Apple', 'Bananas']] set_output( , transform=None)* Set output container. See :ref:`sphx_glr_auto_examples_miscellaneous_plot_set_output.py` for an example on how to use the API. Parameters transform : {\"default\", \"pandas\"}, default=None Configure output of transform and fit_transform . \"default\" : Default output format of a transformer \"pandas\" : DataFrame output None : Transform configuration is unchanged Returns self : estimator instance Estimator instance. set_params( params) Set the parameters of this estimator. The method works on simple estimators as well as on nested objects (such as :class:`~sklearn.pipeline.Pipeline`). The latter have parameters of the form ``<component>__<parameter>`` so that it's possible to update each component of a nested object. Parameters **params : dict Estimator parameters. Returns self : estimator instance Estimator instance. transform(X, sparse=False) Transform transactions into a one-hot encoded NumPy array. Parameters X : list of lists A python list of lists, where the outer list stores the n transactions and the inner list stores the items in each transaction. For example, [['Apple', 'Beer', 'Rice', 'Chicken'], ['Apple', 'Beer', 'Rice'], ['Apple', 'Beer'], ['Apple', 'Bananas'], ['Milk', 'Beer', 'Rice', 'Chicken'], ['Milk', 'Beer', 'Rice'], ['Milk', 'Beer'], ['Apple', 'Bananas']] sparse: bool (default=False) If True, transform will return Compressed Sparse Row matrix instead of the regular one. Returns array : NumPy array [n_transactions, n_unique_items] if sparse=False (default). Compressed Sparse Row matrix otherwise The one-hot encoded boolean array of the input transactions, where the columns represent the unique items found in the input array in alphabetic order. Exact representation depends on the sparse argument For example, array([[True , False, True , True , False, True ], [True , False, True , False, False, True ], [True , False, True , False, False, False], [True , True , False, False, False, False], [False, False, True , True , True , True ], [False, False, True , False, True , True ], [False, False, True , False, True , False], [True , True , False, False, False, False]]) The corresponding column labels are available as self.columns_, e.g., ['Apple', 'Bananas', 'Beer', 'Chicken', 'Milk', 'Rice']","title":"Methods"},{"location":"api_subpackages/mlxtend.preprocessing/#minmax_scaling","text":"minmax_scaling(array, columns, min_val=0, max_val=1) Min max scaling of pandas' DataFrames. Parameters array : pandas DataFrame or NumPy ndarray, shape = [n_rows, n_columns]. columns : array-like, shape = [n_columns] Array-like with column names, e.g., ['col1', 'col2', ...] or column indices [0, 2, 4, ...] min_val : int or float , optional (default= 0 ) minimum value after rescaling. max_val : int or float , optional (default= 1 ) maximum value after rescaling. Returns df_new : pandas DataFrame object. Copy of the array or DataFrame with rescaled columns. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/preprocessing/minmax_scaling/","title":"minmax_scaling"},{"location":"api_subpackages/mlxtend.preprocessing/#one_hot","text":"one_hot(y, num_labels='auto', dtype='float') One-hot encoding of class labels Parameters y : array-like, shape = [n_classlabels] Python list or numpy array consisting of class labels. num_labels : int or 'auto' Number of unique labels in the class label array. Infers the number of unique labels from the input array if set to 'auto'. dtype : str NumPy array type (float, float32, float64) of the output array. Returns ary : numpy.ndarray, shape = [n_classlabels] One-hot encoded array, where each sample is represented as a row vector in the returned array. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/preprocessing/one_hot/","title":"one_hot"},{"location":"api_subpackages/mlxtend.preprocessing/#shuffle_arrays_unison","text":"shuffle_arrays_unison(arrays, random_seed=None) Shuffle NumPy arrays in unison. Parameters arrays : array-like, shape = [n_arrays] A list of NumPy arrays. random_seed : int (default: None) Sets the random state. Returns shuffled_arrays : A list of NumPy arrays after shuffling. Examples ``` >>> import numpy as np >>> from mlxtend.preprocessing import shuffle_arrays_unison >>> X1 = np.array([[1, 2, 3], [4, 5, 6], [7, 8, 9]]) >>> y1 = np.array([1, 2, 3]) >>> X2, y2 = shuffle_arrays_unison(arrays=[X1, y1], random_seed=3) >>> assert(X2.all() == np.array([[4, 5, 6], [1, 2, 3], [7, 8, 9]]).all()) >>> assert(y2.all() == np.array([2, 1, 3]).all()) >>> For more usage examples, please see https://rasbt.github.io/mlxtend/user_guide/preprocessing/shuffle_arrays_unison/ ```","title":"shuffle_arrays_unison"},{"location":"api_subpackages/mlxtend.preprocessing/#standardize","text":"standardize(array, columns=None, ddof=0, return_params=False, params=None) Standardize columns in pandas DataFrames. Parameters array : pandas DataFrame or NumPy ndarray, shape = [n_rows, n_columns]. columns : array-like, shape = [n_columns] (default: None) Array-like with column names, e.g., ['col1', 'col2', ...] or column indices [0, 2, 4, ...] If None, standardizes all columns. ddof : int (default: 0) Delta Degrees of Freedom. The divisor used in calculations is N - ddof, where N represents the number of elements. return_params : dict (default: False) If set to True, a dictionary is returned in addition to the standardized array. The parameter dictionary contains the column means ('avgs') and standard deviations ('stds') of the individual columns. params : dict (default: None) A dictionary with column means and standard deviations as returned by the standardize function if return_params was set to True. If a params dictionary is provided, the standardize function will use these instead of computing them from the current array. Notes If all values in a given column are the same, these values are all set to 0.0 . The standard deviation in the parameters dictionary is consequently set to 1.0 to avoid dividing by zero. Returns df_new : pandas DataFrame object. Copy of the array or DataFrame with standardized columns. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/preprocessing/standardize/","title":"standardize"},{"location":"api_subpackages/mlxtend.regressor/","text":"mlxtend version: 0.23.1 LinearRegression LinearRegression(method='direct', eta=0.01, epochs=50, minibatches=None, random_seed=None, print_progress=0) Ordinary least squares linear regression. Parameters method : string (default: 'direct') For gradient descent-based optimization, use sgd (see minibatch parameter for further options). Otherwise, if direct (default), the analytical method is used. For alternative, numerically more stable solutions, use either qr (QR decomopisition) or svd (Singular Value Decomposition). eta : float (default: 0.01) solver learning rate (between 0.0 and 1.0). Used with method = 'sgd' . (See methods parameter for details) epochs : int (default: 50) Passes over the training dataset. Prior to each epoch, the dataset is shuffled if minibatches > 1 to prevent cycles in stochastic gradient descent. Used with method = 'sgd' . (See methods parameter for details) minibatches : int (default: None) The number of minibatches for gradient-based optimization. If None: Direct method, QR, or SVD method (see method parameter for details) If 1: Gradient Descent learning If len(y): Stochastic Gradient Descent learning If 1 < minibatches < len(y): Minibatch learning random_seed : int (default: None) Set random state for shuffling and initializing the weights. Used in method = 'sgd' . (See methods parameter for details) print_progress : int (default: 0) Prints progress in fitting to stderr if method = 'sgd' . 0: No output 1: Epochs elapsed and cost 2: 1 plus time elapsed 3: 2 plus estimated time until completion Attributes w_ : 2d-array, shape={n_features, 1} Model weights after fitting. b_ : 1d-array, shape={1,} Bias unit after fitting. cost_ : list Sum of squared errors after each epoch; ignored if solver='normal equation' Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/regressor/LinearRegression/ Methods fit(X, y, init_params=True) Learn model from training data. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values. init_params : bool (default: True) Re-initializes model parameters prior to fitting. Set False to continue training with weights from a previous model fitting. Returns self : object get_params(deep=True) Get parameters for this estimator. Parameters deep : boolean, optional If True, will return the parameters for this estimator and contained subobjects that are estimators. Returns params : mapping of string to any Parameter names mapped to their values.' adapted from https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/base.py Author: Gael Varoquaux gael.varoquaux@normalesup.org License: BSD 3 clause predict(X) Predict targets from X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns target_values : array-like, shape = [n_samples] Predicted target values. set_params( params) Set the parameters of this estimator. The method works on simple estimators as well as on nested objects (such as pipelines). The latter have parameters of the form <component>__<parameter> so that it's possible to update each component of a nested object. Returns self adapted from https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/base.py Author: Gael Varoquaux <gael.varoquaux@normalesup.org> License: BSD 3 clause StackingCVRegressor StackingCVRegressor(regressors, meta_regressor, cv=5, shuffle=True, random_state=None, verbose=0, refit=True, use_features_in_secondary=False, store_train_meta_features=False, n_jobs=None, pre_dispatch='2 n_jobs', multi_output=False)* A 'Stacking Cross-Validation' regressor for scikit-learn estimators. Parameters regressors : array-like, shape = [n_regressors] A list of regressors. Invoking the fit method on the StackingCVRegressor will fit clones of these original regressors that will be stored in the class attribute self.regr_ . meta_regressor : object The meta-regressor to be fitted on the ensemble of regressor cv : int, cross-validation generator or iterable, optional (default: 5) Determines the cross-validation splitting strategy. Possible inputs for cv are: - None, to use the default 5-fold cross validation, - integer, to specify the number of folds in a KFold , - An object to be used as a cross-validation generator. - An iterable yielding train, test splits. For integer/None inputs, it will use KFold cross-validation shuffle : bool (default: True) If True, and the cv argument is integer, the training data will be shuffled at fitting stage prior to cross-validation. If the cv argument is a specific cross validation technique, this argument is omitted. random_state : int, RandomState instance or None, optional (default: None) Constrols the randomness of the cv splitter. Used when cv is integer and shuffle=True . New in v0.16.0. verbose : int, optional (default=0) Controls the verbosity of the building process. New in v0.16.0 refit : bool (default: True) Clones the regressors for stacking regression if True (default) or else uses the original ones, which will be refitted on the dataset upon calling the fit method. Setting refit=False is recommended if you are working with estimators that are supporting the scikit-learn fit/predict API interface but are not compatible to scikit-learn's clone function. use_features_in_secondary : bool (default: False) If True, the meta-regressor will be trained both on the predictions of the original regressors and the original dataset. If False, the meta-regressor will be trained only on the predictions of the original regressors. store_train_meta_features : bool (default: False) If True, the meta-features computed from the training data used for fitting the meta-regressor stored in the self.train_meta_features_ array, which can be accessed after calling fit . n_jobs : int or None, optional (default=None) The number of CPUs to use to do the computation. None means 1 unless in a :obj: joblib.parallel_backend context. -1 means using all processors. See :term: Glossary <n_jobs> for more details. New in v0.16.0. pre_dispatch : int, or string, optional Controls the number of jobs that get dispatched during parallel execution. Reducing this number can be useful to avoid an explosion of memory consumption when more jobs get dispatched than CPUs can process. This parameter can be: - None, in which case all the jobs are immediately created and spawned. Use this for lightweight and fast-running jobs, to avoid delays due to on-demand spawning of the jobs - An int, giving the exact number of total jobs that are spawned - A string, giving an expression as a function of n_jobs, as in '2*n_jobs' multi_output : bool (default: False) If True, allow multi-output targets, but forbid nan or inf values. If False, y will be checked to be a vector. (New in v0.19.0.) Attributes train_meta_features : numpy array, shape = [n_samples, n_regressors] meta-features for training data, where n_samples is the number of samples in training data and len(self.regressors) is the number of regressors. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/regressor/StackingCVRegressor/ Methods fit(X, y, groups=None, sample_weight=None) Fit ensemble regressors and the meta-regressor. Parameters X : numpy array, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : numpy array, shape = [n_samples] or [n_samples, n_targets] Target values. Multiple targets are supported only if self.multi_output is True. groups : numpy array/None, shape = [n_samples] The group that each sample belongs to. This is used by specific folding strategies such as GroupKFold() sample_weight : array-like, shape = [n_samples], optional Sample weights passed as sample_weights to each regressor in the regressors list as well as the meta_regressor. Raises error if some regressor does not support sample_weight in the fit() method. Returns self : object fit_transform(X, y=None, fit_params) Fit to data, then transform it. Fits transformer to `X` and `y` with optional parameters `fit_params` and returns a transformed version of `X`. Parameters X : array-like of shape (n_samples, n_features) Input samples. y : array-like of shape (n_samples,) or (n_samples, n_outputs), default=None Target values (None for unsupervised transformations). **fit_params : dict Additional fit parameters. Returns X_new : ndarray array of shape (n_samples, n_features_new) Transformed array. get_params(deep=True) Get parameters for this estimator. Parameters deep : bool, default=True If True, will return the parameters for this estimator and contained subobjects that are estimators. Returns params : dict Parameter names mapped to their values. predict(X) Predict target values for X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns y_target : array-like, shape = [n_samples] or [n_samples, n_targets] Predicted target values. predict_meta_features(X) Get meta-features of test-data. Parameters X : numpy array, shape = [n_samples, n_features] Test vectors, where n_samples is the number of samples and n_features is the number of features. Returns meta-features : numpy array, shape = [n_samples, len(self.regressors)] meta-features for test data, where n_samples is the number of samples in test data and len(self.regressors) is the number of regressors. If self.multi_output is True, then the number of columns is len(self.regressors) * n_targets. score(X, y, sample_weight=None) Return the coefficient of determination of the prediction. The coefficient of determination :math:`R^2` is defined as :math:`(1 - \\frac{u}{v})`, where :math:`u` is the residual sum of squares ((y_true - y_pred)** 2).sum() and :math: v is the total sum of squares ((y_true - y_true.mean()) ** 2).sum() . The best possible score is 1.0 and it can be negative (because the model can be arbitrarily worse). A constant model that always predicts the expected value of y , disregarding the input features, would get a :math: R^2 score of 0.0. Parameters X : array-like of shape (n_samples, n_features) Test samples. For some estimators this may be a precomputed kernel matrix or a list of generic objects instead with shape (n_samples, n_samples_fitted) , where n_samples_fitted is the number of samples used in the fitting for the estimator. y : array-like of shape (n_samples,) or (n_samples, n_outputs) True values for X . sample_weight : array-like of shape (n_samples,), default=None Sample weights. Returns score : float :math: R^2 of self.predict(X) w.r.t. y . Notes The :math: R^2 score used when calling score on a regressor uses multioutput='uniform_average' from version 0.23 to keep consistent with default value of :func: ~sklearn.metrics.r2_score . This influences the score method of all the multioutput regressors (except for :class: ~sklearn.multioutput.MultiOutputRegressor ). set_output( , transform=None)* Set output container. See :ref:`sphx_glr_auto_examples_miscellaneous_plot_set_output.py` for an example on how to use the API. Parameters transform : {\"default\", \"pandas\"}, default=None Configure output of transform and fit_transform . \"default\" : Default output format of a transformer \"pandas\" : DataFrame output None : Transform configuration is unchanged Returns self : estimator instance Estimator instance. set_params( params) Set the parameters of this estimator. Valid parameter keys can be listed with ``get_params()``. Returns self Properties named_regressors Returns List of named estimator tuples, like [('svc', SVC(...))] StackingRegressor StackingRegressor(regressors, meta_regressor, verbose=0, use_features_in_secondary=False, store_train_meta_features=False, refit=True, multi_output=False) A Stacking regressor for scikit-learn estimators for regression. Parameters regressors : array-like, shape = [n_regressors] A list of regressors. Invoking the fit method on the StackingRegressor will fit clones of those original regressors that will be stored in the class attribute self.regr_ . meta_regressor : object The meta-regressor to be fitted on the ensemble of regressors verbose : int, optional (default=0) Controls the verbosity of the building process. - verbose=0 (default): Prints nothing - verbose=1 : Prints the number & name of the regressor being fitted - verbose=2 : Prints info about the parameters of the regressor being fitted - verbose>2 : Changes verbose param of the underlying regressor to self.verbose - 2 use_features_in_secondary : bool (default: False) If True, the meta-regressor will be trained both on the predictions of the original regressors and the original dataset. If False, the meta-regressor will be trained only on the predictions of the original regressors. store_train_meta_features : bool (default: False) If True, the meta-features computed from the training data used for fitting the meta-regressor stored in the self.train_meta_features_ array, which can be accessed after calling fit . Attributes regr_ : list, shape=[n_regressors] Fitted regressors (clones of the original regressors) meta_regr_ : estimator Fitted meta-regressor (clone of the original meta-estimator) coef_ : array-like, shape = [n_features] Model coefficients of the fitted meta-estimator intercept_ : float Intercept of the fitted meta-estimator train_meta_features : numpy array, shape = [n_samples, len(self.regressors)] meta-features for training data, where n_samples is the number of samples in training data and len(self.regressors) is the number of regressors. refit : bool (default: True) Clones the regressors for stacking regression if True (default) or else uses the original ones, which will be refitted on the dataset upon calling the fit method. Setting refit=False is recommended if you are working with estimators that are supporting the scikit-learn fit/predict API interface but are not compatible to scikit-learn's clone function. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/regressor/StackingRegressor/ Methods fit(X, y, sample_weight=None) Learn weight coefficients from training data for each regressor. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : numpy array, shape = [n_samples] or [n_samples, n_targets] Target values. Multiple targets are supported only if self.multi_output is True. sample_weight : array-like, shape = [n_samples], optional Sample weights passed as sample_weights to each regressor in the regressors list as well as the meta_regressor. Raises error if some regressor does not support sample_weight in the fit() method. Returns self : object fit_transform(X, y=None, fit_params) Fit to data, then transform it. Fits transformer to `X` and `y` with optional parameters `fit_params` and returns a transformed version of `X`. Parameters X : array-like of shape (n_samples, n_features) Input samples. y : array-like of shape (n_samples,) or (n_samples, n_outputs), default=None Target values (None for unsupervised transformations). **fit_params : dict Additional fit parameters. Returns X_new : ndarray array of shape (n_samples, n_features_new) Transformed array. get_params(deep=True) Return estimator parameter names for GridSearch support. predict(X) Predict target values for X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns y_target : array-like, shape = [n_samples] or [n_samples, n_targets] Predicted target values. predict_meta_features(X) Get meta-features of test-data. Parameters X : numpy array, shape = [n_samples, n_features] Test vectors, where n_samples is the number of samples and n_features is the number of features. Returns meta-features : numpy array, shape = [n_samples, len(self.regressors)] meta-features for test data, where n_samples is the number of samples in test data and len(self.regressors) is the number of regressors. If self.multi_output is True, then the number of columns is len(self.regressors) * n_targets score(X, y, sample_weight=None) Return the coefficient of determination of the prediction. The coefficient of determination :math:`R^2` is defined as :math:`(1 - \\frac{u}{v})`, where :math:`u` is the residual sum of squares ((y_true - y_pred)** 2).sum() and :math: v is the total sum of squares ((y_true - y_true.mean()) ** 2).sum() . The best possible score is 1.0 and it can be negative (because the model can be arbitrarily worse). A constant model that always predicts the expected value of y , disregarding the input features, would get a :math: R^2 score of 0.0. Parameters X : array-like of shape (n_samples, n_features) Test samples. For some estimators this may be a precomputed kernel matrix or a list of generic objects instead with shape (n_samples, n_samples_fitted) , where n_samples_fitted is the number of samples used in the fitting for the estimator. y : array-like of shape (n_samples,) or (n_samples, n_outputs) True values for X . sample_weight : array-like of shape (n_samples,), default=None Sample weights. Returns score : float :math: R^2 of self.predict(X) w.r.t. y . Notes The :math: R^2 score used when calling score on a regressor uses multioutput='uniform_average' from version 0.23 to keep consistent with default value of :func: ~sklearn.metrics.r2_score . This influences the score method of all the multioutput regressors (except for :class: ~sklearn.multioutput.MultiOutputRegressor ). set_output( , transform=None)* Set output container. See :ref:`sphx_glr_auto_examples_miscellaneous_plot_set_output.py` for an example on how to use the API. Parameters transform : {\"default\", \"pandas\"}, default=None Configure output of transform and fit_transform . \"default\" : Default output format of a transformer \"pandas\" : DataFrame output None : Transform configuration is unchanged Returns self : estimator instance Estimator instance. set_params( params) Set the parameters of this estimator. Valid parameter keys can be listed with ``get_params()``. Returns self Properties coef_ None intercept_ None named_regressors None","title":"Mlxtend.regressor"},{"location":"api_subpackages/mlxtend.regressor/#linearregression","text":"LinearRegression(method='direct', eta=0.01, epochs=50, minibatches=None, random_seed=None, print_progress=0) Ordinary least squares linear regression. Parameters method : string (default: 'direct') For gradient descent-based optimization, use sgd (see minibatch parameter for further options). Otherwise, if direct (default), the analytical method is used. For alternative, numerically more stable solutions, use either qr (QR decomopisition) or svd (Singular Value Decomposition). eta : float (default: 0.01) solver learning rate (between 0.0 and 1.0). Used with method = 'sgd' . (See methods parameter for details) epochs : int (default: 50) Passes over the training dataset. Prior to each epoch, the dataset is shuffled if minibatches > 1 to prevent cycles in stochastic gradient descent. Used with method = 'sgd' . (See methods parameter for details) minibatches : int (default: None) The number of minibatches for gradient-based optimization. If None: Direct method, QR, or SVD method (see method parameter for details) If 1: Gradient Descent learning If len(y): Stochastic Gradient Descent learning If 1 < minibatches < len(y): Minibatch learning random_seed : int (default: None) Set random state for shuffling and initializing the weights. Used in method = 'sgd' . (See methods parameter for details) print_progress : int (default: 0) Prints progress in fitting to stderr if method = 'sgd' . 0: No output 1: Epochs elapsed and cost 2: 1 plus time elapsed 3: 2 plus estimated time until completion Attributes w_ : 2d-array, shape={n_features, 1} Model weights after fitting. b_ : 1d-array, shape={1,} Bias unit after fitting. cost_ : list Sum of squared errors after each epoch; ignored if solver='normal equation' Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/regressor/LinearRegression/","title":"LinearRegression"},{"location":"api_subpackages/mlxtend.regressor/#methods","text":"fit(X, y, init_params=True) Learn model from training data. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values. init_params : bool (default: True) Re-initializes model parameters prior to fitting. Set False to continue training with weights from a previous model fitting. Returns self : object get_params(deep=True) Get parameters for this estimator. Parameters deep : boolean, optional If True, will return the parameters for this estimator and contained subobjects that are estimators. Returns params : mapping of string to any Parameter names mapped to their values.' adapted from https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/base.py Author: Gael Varoquaux gael.varoquaux@normalesup.org License: BSD 3 clause predict(X) Predict targets from X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns target_values : array-like, shape = [n_samples] Predicted target values. set_params( params) Set the parameters of this estimator. The method works on simple estimators as well as on nested objects (such as pipelines). The latter have parameters of the form <component>__<parameter> so that it's possible to update each component of a nested object. Returns self adapted from https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/base.py Author: Gael Varoquaux <gael.varoquaux@normalesup.org> License: BSD 3 clause","title":"Methods"},{"location":"api_subpackages/mlxtend.regressor/#stackingcvregressor","text":"StackingCVRegressor(regressors, meta_regressor, cv=5, shuffle=True, random_state=None, verbose=0, refit=True, use_features_in_secondary=False, store_train_meta_features=False, n_jobs=None, pre_dispatch='2 n_jobs', multi_output=False)* A 'Stacking Cross-Validation' regressor for scikit-learn estimators. Parameters regressors : array-like, shape = [n_regressors] A list of regressors. Invoking the fit method on the StackingCVRegressor will fit clones of these original regressors that will be stored in the class attribute self.regr_ . meta_regressor : object The meta-regressor to be fitted on the ensemble of regressor cv : int, cross-validation generator or iterable, optional (default: 5) Determines the cross-validation splitting strategy. Possible inputs for cv are: - None, to use the default 5-fold cross validation, - integer, to specify the number of folds in a KFold , - An object to be used as a cross-validation generator. - An iterable yielding train, test splits. For integer/None inputs, it will use KFold cross-validation shuffle : bool (default: True) If True, and the cv argument is integer, the training data will be shuffled at fitting stage prior to cross-validation. If the cv argument is a specific cross validation technique, this argument is omitted. random_state : int, RandomState instance or None, optional (default: None) Constrols the randomness of the cv splitter. Used when cv is integer and shuffle=True . New in v0.16.0. verbose : int, optional (default=0) Controls the verbosity of the building process. New in v0.16.0 refit : bool (default: True) Clones the regressors for stacking regression if True (default) or else uses the original ones, which will be refitted on the dataset upon calling the fit method. Setting refit=False is recommended if you are working with estimators that are supporting the scikit-learn fit/predict API interface but are not compatible to scikit-learn's clone function. use_features_in_secondary : bool (default: False) If True, the meta-regressor will be trained both on the predictions of the original regressors and the original dataset. If False, the meta-regressor will be trained only on the predictions of the original regressors. store_train_meta_features : bool (default: False) If True, the meta-features computed from the training data used for fitting the meta-regressor stored in the self.train_meta_features_ array, which can be accessed after calling fit . n_jobs : int or None, optional (default=None) The number of CPUs to use to do the computation. None means 1 unless in a :obj: joblib.parallel_backend context. -1 means using all processors. See :term: Glossary <n_jobs> for more details. New in v0.16.0. pre_dispatch : int, or string, optional Controls the number of jobs that get dispatched during parallel execution. Reducing this number can be useful to avoid an explosion of memory consumption when more jobs get dispatched than CPUs can process. This parameter can be: - None, in which case all the jobs are immediately created and spawned. Use this for lightweight and fast-running jobs, to avoid delays due to on-demand spawning of the jobs - An int, giving the exact number of total jobs that are spawned - A string, giving an expression as a function of n_jobs, as in '2*n_jobs' multi_output : bool (default: False) If True, allow multi-output targets, but forbid nan or inf values. If False, y will be checked to be a vector. (New in v0.19.0.) Attributes train_meta_features : numpy array, shape = [n_samples, n_regressors] meta-features for training data, where n_samples is the number of samples in training data and len(self.regressors) is the number of regressors. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/regressor/StackingCVRegressor/","title":"StackingCVRegressor"},{"location":"api_subpackages/mlxtend.regressor/#methods_1","text":"fit(X, y, groups=None, sample_weight=None) Fit ensemble regressors and the meta-regressor. Parameters X : numpy array, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : numpy array, shape = [n_samples] or [n_samples, n_targets] Target values. Multiple targets are supported only if self.multi_output is True. groups : numpy array/None, shape = [n_samples] The group that each sample belongs to. This is used by specific folding strategies such as GroupKFold() sample_weight : array-like, shape = [n_samples], optional Sample weights passed as sample_weights to each regressor in the regressors list as well as the meta_regressor. Raises error if some regressor does not support sample_weight in the fit() method. Returns self : object fit_transform(X, y=None, fit_params) Fit to data, then transform it. Fits transformer to `X` and `y` with optional parameters `fit_params` and returns a transformed version of `X`. Parameters X : array-like of shape (n_samples, n_features) Input samples. y : array-like of shape (n_samples,) or (n_samples, n_outputs), default=None Target values (None for unsupervised transformations). **fit_params : dict Additional fit parameters. Returns X_new : ndarray array of shape (n_samples, n_features_new) Transformed array. get_params(deep=True) Get parameters for this estimator. Parameters deep : bool, default=True If True, will return the parameters for this estimator and contained subobjects that are estimators. Returns params : dict Parameter names mapped to their values. predict(X) Predict target values for X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns y_target : array-like, shape = [n_samples] or [n_samples, n_targets] Predicted target values. predict_meta_features(X) Get meta-features of test-data. Parameters X : numpy array, shape = [n_samples, n_features] Test vectors, where n_samples is the number of samples and n_features is the number of features. Returns meta-features : numpy array, shape = [n_samples, len(self.regressors)] meta-features for test data, where n_samples is the number of samples in test data and len(self.regressors) is the number of regressors. If self.multi_output is True, then the number of columns is len(self.regressors) * n_targets. score(X, y, sample_weight=None) Return the coefficient of determination of the prediction. The coefficient of determination :math:`R^2` is defined as :math:`(1 - \\frac{u}{v})`, where :math:`u` is the residual sum of squares ((y_true - y_pred)** 2).sum() and :math: v is the total sum of squares ((y_true - y_true.mean()) ** 2).sum() . The best possible score is 1.0 and it can be negative (because the model can be arbitrarily worse). A constant model that always predicts the expected value of y , disregarding the input features, would get a :math: R^2 score of 0.0. Parameters X : array-like of shape (n_samples, n_features) Test samples. For some estimators this may be a precomputed kernel matrix or a list of generic objects instead with shape (n_samples, n_samples_fitted) , where n_samples_fitted is the number of samples used in the fitting for the estimator. y : array-like of shape (n_samples,) or (n_samples, n_outputs) True values for X . sample_weight : array-like of shape (n_samples,), default=None Sample weights. Returns score : float :math: R^2 of self.predict(X) w.r.t. y . Notes The :math: R^2 score used when calling score on a regressor uses multioutput='uniform_average' from version 0.23 to keep consistent with default value of :func: ~sklearn.metrics.r2_score . This influences the score method of all the multioutput regressors (except for :class: ~sklearn.multioutput.MultiOutputRegressor ). set_output( , transform=None)* Set output container. See :ref:`sphx_glr_auto_examples_miscellaneous_plot_set_output.py` for an example on how to use the API. Parameters transform : {\"default\", \"pandas\"}, default=None Configure output of transform and fit_transform . \"default\" : Default output format of a transformer \"pandas\" : DataFrame output None : Transform configuration is unchanged Returns self : estimator instance Estimator instance. set_params( params) Set the parameters of this estimator. Valid parameter keys can be listed with ``get_params()``. Returns self","title":"Methods"},{"location":"api_subpackages/mlxtend.regressor/#properties","text":"named_regressors Returns List of named estimator tuples, like [('svc', SVC(...))]","title":"Properties"},{"location":"api_subpackages/mlxtend.regressor/#stackingregressor","text":"StackingRegressor(regressors, meta_regressor, verbose=0, use_features_in_secondary=False, store_train_meta_features=False, refit=True, multi_output=False) A Stacking regressor for scikit-learn estimators for regression. Parameters regressors : array-like, shape = [n_regressors] A list of regressors. Invoking the fit method on the StackingRegressor will fit clones of those original regressors that will be stored in the class attribute self.regr_ . meta_regressor : object The meta-regressor to be fitted on the ensemble of regressors verbose : int, optional (default=0) Controls the verbosity of the building process. - verbose=0 (default): Prints nothing - verbose=1 : Prints the number & name of the regressor being fitted - verbose=2 : Prints info about the parameters of the regressor being fitted - verbose>2 : Changes verbose param of the underlying regressor to self.verbose - 2 use_features_in_secondary : bool (default: False) If True, the meta-regressor will be trained both on the predictions of the original regressors and the original dataset. If False, the meta-regressor will be trained only on the predictions of the original regressors. store_train_meta_features : bool (default: False) If True, the meta-features computed from the training data used for fitting the meta-regressor stored in the self.train_meta_features_ array, which can be accessed after calling fit . Attributes regr_ : list, shape=[n_regressors] Fitted regressors (clones of the original regressors) meta_regr_ : estimator Fitted meta-regressor (clone of the original meta-estimator) coef_ : array-like, shape = [n_features] Model coefficients of the fitted meta-estimator intercept_ : float Intercept of the fitted meta-estimator train_meta_features : numpy array, shape = [n_samples, len(self.regressors)] meta-features for training data, where n_samples is the number of samples in training data and len(self.regressors) is the number of regressors. refit : bool (default: True) Clones the regressors for stacking regression if True (default) or else uses the original ones, which will be refitted on the dataset upon calling the fit method. Setting refit=False is recommended if you are working with estimators that are supporting the scikit-learn fit/predict API interface but are not compatible to scikit-learn's clone function. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/regressor/StackingRegressor/","title":"StackingRegressor"},{"location":"api_subpackages/mlxtend.regressor/#methods_2","text":"fit(X, y, sample_weight=None) Learn weight coefficients from training data for each regressor. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : numpy array, shape = [n_samples] or [n_samples, n_targets] Target values. Multiple targets are supported only if self.multi_output is True. sample_weight : array-like, shape = [n_samples], optional Sample weights passed as sample_weights to each regressor in the regressors list as well as the meta_regressor. Raises error if some regressor does not support sample_weight in the fit() method. Returns self : object fit_transform(X, y=None, fit_params) Fit to data, then transform it. Fits transformer to `X` and `y` with optional parameters `fit_params` and returns a transformed version of `X`. Parameters X : array-like of shape (n_samples, n_features) Input samples. y : array-like of shape (n_samples,) or (n_samples, n_outputs), default=None Target values (None for unsupervised transformations). **fit_params : dict Additional fit parameters. Returns X_new : ndarray array of shape (n_samples, n_features_new) Transformed array. get_params(deep=True) Return estimator parameter names for GridSearch support. predict(X) Predict target values for X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns y_target : array-like, shape = [n_samples] or [n_samples, n_targets] Predicted target values. predict_meta_features(X) Get meta-features of test-data. Parameters X : numpy array, shape = [n_samples, n_features] Test vectors, where n_samples is the number of samples and n_features is the number of features. Returns meta-features : numpy array, shape = [n_samples, len(self.regressors)] meta-features for test data, where n_samples is the number of samples in test data and len(self.regressors) is the number of regressors. If self.multi_output is True, then the number of columns is len(self.regressors) * n_targets score(X, y, sample_weight=None) Return the coefficient of determination of the prediction. The coefficient of determination :math:`R^2` is defined as :math:`(1 - \\frac{u}{v})`, where :math:`u` is the residual sum of squares ((y_true - y_pred)** 2).sum() and :math: v is the total sum of squares ((y_true - y_true.mean()) ** 2).sum() . The best possible score is 1.0 and it can be negative (because the model can be arbitrarily worse). A constant model that always predicts the expected value of y , disregarding the input features, would get a :math: R^2 score of 0.0. Parameters X : array-like of shape (n_samples, n_features) Test samples. For some estimators this may be a precomputed kernel matrix or a list of generic objects instead with shape (n_samples, n_samples_fitted) , where n_samples_fitted is the number of samples used in the fitting for the estimator. y : array-like of shape (n_samples,) or (n_samples, n_outputs) True values for X . sample_weight : array-like of shape (n_samples,), default=None Sample weights. Returns score : float :math: R^2 of self.predict(X) w.r.t. y . Notes The :math: R^2 score used when calling score on a regressor uses multioutput='uniform_average' from version 0.23 to keep consistent with default value of :func: ~sklearn.metrics.r2_score . This influences the score method of all the multioutput regressors (except for :class: ~sklearn.multioutput.MultiOutputRegressor ). set_output( , transform=None)* Set output container. See :ref:`sphx_glr_auto_examples_miscellaneous_plot_set_output.py` for an example on how to use the API. Parameters transform : {\"default\", \"pandas\"}, default=None Configure output of transform and fit_transform . \"default\" : Default output format of a transformer \"pandas\" : DataFrame output None : Transform configuration is unchanged Returns self : estimator instance Estimator instance. set_params( params) Set the parameters of this estimator. Valid parameter keys can be listed with ``get_params()``. Returns self","title":"Methods"},{"location":"api_subpackages/mlxtend.regressor/#properties_1","text":"coef_ None intercept_ None named_regressors None","title":"Properties"},{"location":"api_subpackages/mlxtend.text/","text":"mlxtend version: 0.23.1 generalize_names generalize_names(name, output_sep=' ', firstname_output_letters=1) Generalize a person's first and last name. Returns a person's name in the format <last_name><separator><firstname letter(s)> (all lowercase) Parameters name : str Name of the player output_sep : str (default: ' ') String for separating last name and first name in the output. firstname_output_letters : int Number of letters in the abbreviated first name. Returns gen_name : str The generalized name. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/text/generalize_names/ generalize_names_duplcheck generalize_names_duplcheck(df, col_name) Generalizes names and removes duplicates. Applies mlxtend.text.generalize_names to a DataFrame with 1 first name letter by default and uses more first name letters if duplicates are detected. Parameters df : pandas.DataFrame DataFrame that contains a column where generalize_names should be applied. col_name : str Name of the DataFrame column where generalize_names function should be applied to. Returns df_new : str New DataFrame object where generalize_names function has been applied without duplicates. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/text/generalize_names_duplcheck/ tokenizer_emoticons tokenizer_emoticons(text) Return emoticons from text Examples >>> tokenizer_emoticons('</a>This :) is :( a test :-)!') [':)', ':(', ':-)'] For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/text/tokenizer_emoticons/ tokenizer_words_and_emoticons tokenizer_words_and_emoticons(text) Convert text to lowercase words and emoticons. Examples >>> tokenizer_words_and_emoticons('</a>This :) is :( a test :-)!') ['this', 'is', 'a', 'test', ':)', ':(', ':-)'] For more usage examples, please see https://rasbt.github.io/mlxtend/user_guide/text/tokenizer_words_and_emoticons/","title":"Mlxtend.text"},{"location":"api_subpackages/mlxtend.text/#generalize_names","text":"generalize_names(name, output_sep=' ', firstname_output_letters=1) Generalize a person's first and last name. Returns a person's name in the format <last_name><separator><firstname letter(s)> (all lowercase) Parameters name : str Name of the player output_sep : str (default: ' ') String for separating last name and first name in the output. firstname_output_letters : int Number of letters in the abbreviated first name. Returns gen_name : str The generalized name. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/text/generalize_names/","title":"generalize_names"},{"location":"api_subpackages/mlxtend.text/#generalize_names_duplcheck","text":"generalize_names_duplcheck(df, col_name) Generalizes names and removes duplicates. Applies mlxtend.text.generalize_names to a DataFrame with 1 first name letter by default and uses more first name letters if duplicates are detected. Parameters df : pandas.DataFrame DataFrame that contains a column where generalize_names should be applied. col_name : str Name of the DataFrame column where generalize_names function should be applied to. Returns df_new : str New DataFrame object where generalize_names function has been applied without duplicates. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/text/generalize_names_duplcheck/","title":"generalize_names_duplcheck"},{"location":"api_subpackages/mlxtend.text/#tokenizer_emoticons","text":"tokenizer_emoticons(text) Return emoticons from text Examples >>> tokenizer_emoticons('</a>This :) is :( a test :-)!') [':)', ':(', ':-)'] For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/text/tokenizer_emoticons/","title":"tokenizer_emoticons"},{"location":"api_subpackages/mlxtend.text/#tokenizer_words_and_emoticons","text":"tokenizer_words_and_emoticons(text) Convert text to lowercase words and emoticons. Examples >>> tokenizer_words_and_emoticons('</a>This :) is :( a test :-)!') ['this', 'is', 'a', 'test', ':)', ':(', ':-)'] For more usage examples, please see https://rasbt.github.io/mlxtend/user_guide/text/tokenizer_words_and_emoticons/","title":"tokenizer_words_and_emoticons"},{"location":"api_subpackages/mlxtend.utils/","text":"mlxtend version: 0.23.1 Counter Counter(stderr=False, start_newline=True, precision=0, name=None) Class to display the progress of for-loop iterators. Parameters stderr : bool (default: True) Prints output to sys.stderr if True; uses sys.stdout otherwise. start_newline : bool (default: True) Prepends a new line to the counter, which prevents overwriting counters if multiple counters are printed in succession. precision: int (default: 0) Sets the number of decimal places when displaying the time elapsed in seconds. name : string (default: None) Prepends the specified name before the counter to allow distinguishing between multiple counters. Attributes curr_iter : int The current iteration. start_time : float The system's time in seconds when the Counter was initialized. end_time : float The system's time in seconds when the Counter was last updated. Examples >>> cnt = Counter() >>> for i in range(20): ... # do some computation ... time.sleep(0.1) ... cnt.update() 20 iter | 2 sec >>> print('The counter was initialized.' ' %d seconds ago.' % (time.time() - cnt.start_time)) The counter was initialized 2 seconds ago >>> print('The counter was last updated' ' %d seconds ago.' % (time.time() - cnt.end_time)) The counter was last updated 0 seconds ago. For more usage examples, please see https://rasbt.github.io/mlxtend/user_guide/utils/Counter/ Methods update() Print current iteration and time elapsed. assert_raises assert_raises(exception_type, message, func, args, * kwargs) Check that an exception is raised with a specific message Parameters exception_type : exception The exception that should be raised message : str (default: None) The error message that should be raised. Ignored if False or None. func : callable The function that raises the exception *args : positional arguments to func . **kwargs : keyword arguments to func check_Xy check_Xy(X, y, y_int=True) None format_kwarg_dictionaries format_kwarg_dictionaries(default_kwargs=None, user_kwargs=None, protected_keys=None) Function to combine default and user specified kwargs dictionaries Parameters default_kwargs : dict, optional Default kwargs (default is None). user_kwargs : dict, optional User specified kwargs (default is None). protected_keys : array_like, optional Sequence of keys to be removed from the returned dictionary (default is None). Returns formatted_kwargs : dict Formatted kwargs dictionary.","title":"Mlxtend.utils"},{"location":"api_subpackages/mlxtend.utils/#counter","text":"Counter(stderr=False, start_newline=True, precision=0, name=None) Class to display the progress of for-loop iterators. Parameters stderr : bool (default: True) Prints output to sys.stderr if True; uses sys.stdout otherwise. start_newline : bool (default: True) Prepends a new line to the counter, which prevents overwriting counters if multiple counters are printed in succession. precision: int (default: 0) Sets the number of decimal places when displaying the time elapsed in seconds. name : string (default: None) Prepends the specified name before the counter to allow distinguishing between multiple counters. Attributes curr_iter : int The current iteration. start_time : float The system's time in seconds when the Counter was initialized. end_time : float The system's time in seconds when the Counter was last updated. Examples >>> cnt = Counter() >>> for i in range(20): ... # do some computation ... time.sleep(0.1) ... cnt.update() 20 iter | 2 sec >>> print('The counter was initialized.' ' %d seconds ago.' % (time.time() - cnt.start_time)) The counter was initialized 2 seconds ago >>> print('The counter was last updated' ' %d seconds ago.' % (time.time() - cnt.end_time)) The counter was last updated 0 seconds ago. For more usage examples, please see https://rasbt.github.io/mlxtend/user_guide/utils/Counter/","title":"Counter"},{"location":"api_subpackages/mlxtend.utils/#methods","text":"update() Print current iteration and time elapsed.","title":"Methods"},{"location":"api_subpackages/mlxtend.utils/#assert_raises","text":"assert_raises(exception_type, message, func, args, * kwargs) Check that an exception is raised with a specific message Parameters exception_type : exception The exception that should be raised message : str (default: None) The error message that should be raised. Ignored if False or None. func : callable The function that raises the exception *args : positional arguments to func . **kwargs : keyword arguments to func","title":"assert_raises"},{"location":"api_subpackages/mlxtend.utils/#check_xy","text":"check_Xy(X, y, y_int=True) None","title":"check_Xy"},{"location":"api_subpackages/mlxtend.utils/#format_kwarg_dictionaries","text":"format_kwarg_dictionaries(default_kwargs=None, user_kwargs=None, protected_keys=None) Function to combine default and user specified kwargs dictionaries Parameters default_kwargs : dict, optional Default kwargs (default is None). user_kwargs : dict, optional User specified kwargs (default is None). protected_keys : array_like, optional Sequence of keys to be removed from the returned dictionary (default is None). Returns formatted_kwargs : dict Formatted kwargs dictionary.","title":"format_kwarg_dictionaries"},{"location":"user_guide/classifier/Adaline/","text":"Adaline: Adaptive Linear Neuron Classifier An implementation of the ADAptive LInear NEuron, Adaline, for binary classification tasks. from mlxtend.classifier import Adaline Overview An illustration of the ADAptive LInear NEuron (Adaline) -- a single-layer artificial linear neuron with a threshold unit: The Adaline classifier is closely related to the Ordinary Least Squares (OLS) Linear Regression algorithm; in OLS regression we find the line (or hyperplane) that minimizes the vertical offsets. Or in other words, we define the best-fitting line as the line that minimizes the sum of squared errors (SSE) or mean squared error (MSE) between our target variable (y) and our predicted output over all samples i in our dataset of size n . SSE = \\sum_i (\\text{target}^{(i)} - \\text{output}^{(i)})^2 MSE = \\frac{1}{n} \\times SSE LinearRegression implements a linear regression model for performing ordinary least squares regression, and in Adaline, we add a threshold function g(\\cdot) to convert the continuous outcome to a categorical class label: $$y = g({z}) = \\begin{cases} 1 & \\text{if z $\\ge$ 0}\\\\ -1 & \\text{otherwise}. \\end{cases} $$ An Adaline model can be trained by one of the following three approaches: Normal Equations Gradient Descent Stochastic Gradient Descent Normal Equations (closed-form solution) The closed-form solution should be preferred for \"smaller\" datasets where calculating (a \"costly\") matrix inverse is not a concern. For very large datasets, or datasets where the inverse of [X^T X] may not exist (the matrix is non-invertible or singular, e.g., in case of perfect multicollinearity), the gradient descent or stochastic gradient descent approaches are to be preferred. The linear function (linear regression model) is defined as: z = w_0x_0 + w_1x_1 + ... + w_mx_m = \\sum_{j=0}^{m} w_j x_j = \\mathbf{w}^T\\mathbf{x} where y is the response variable, \\mathbf{x} is an m -dimensional sample vector, and \\mathbf{w} is the weight vector (vector of coefficients). Note that w_0 represents the y-axis intercept of the model and therefore x_0=1 . Using the closed-form solution (normal equation), we compute the weights of the model as follows: \\mathbf{w} = (\\mathbf{X}^T\\mathbf{X})^{-1}\\mathbf{X}^Ty Gradient Descent (GD) and Stochastic Gradient Descent (SGD) In the current implementation, the Adaline model is learned via Gradient Descent or Stochastic Gradient Descent. See Gradient Descent and Stochastic Gradient Descent and Deriving the Gradient Descent Rule for Linear Regression and Adaline for details. Random shuffling is implemented as: for one or more epochs randomly shuffle samples in the training set for training sample i compute gradients and perform weight updates References B. Widrow, M. E. Hoff, et al. Adaptive switching circuits . 1960. Example 1 - Closed Form Solution from mlxtend.data import iris_data from mlxtend.plotting import plot_decision_regions from mlxtend.classifier import Adaline import matplotlib.pyplot as plt # Loading Data X, y = iris_data() X = X[:, [0, 3]] # sepal length and petal width X = X[0:100] # class 0 and class 1 y = y[0:100] # class 0 and class 1 # standardize X[:,0] = (X[:,0] - X[:,0].mean()) / X[:,0].std() X[:,1] = (X[:,1] - X[:,1].mean()) / X[:,1].std() ada = Adaline(epochs=30, eta=0.01, minibatches=None, random_seed=1) ada.fit(X, y) plot_decision_regions(X, y, clf=ada) plt.title('Adaline - Closed Form') plt.show() Example 2 - Gradient Descent from mlxtend.data import iris_data from mlxtend.plotting import plot_decision_regions from mlxtend.classifier import Adaline import matplotlib.pyplot as plt # Loading Data X, y = iris_data() X = X[:, [0, 3]] # sepal length and petal width X = X[0:100] # class 0 and class 1 y = y[0:100] # class 0 and class 1 # standardize X[:,0] = (X[:,0] - X[:,0].mean()) / X[:,0].std() X[:,1] = (X[:,1] - X[:,1].mean()) / X[:,1].std() ada = Adaline(epochs=30, eta=0.01, minibatches=1, # for Gradient Descent Learning random_seed=1, print_progress=3) ada.fit(X, y) plot_decision_regions(X, y, clf=ada) plt.title('Adaline - Gradient Descent') plt.show() plt.plot(range(len(ada.cost_)), ada.cost_) plt.xlabel('Iterations') plt.ylabel('Cost') Iteration: 30/30 | Cost 3.79 | Elapsed: 0:00:00 | ETA: 0:00:00 Text(0, 0.5, 'Cost') Example 3 - Stochastic Gradient Descent from mlxtend.data import iris_data from mlxtend.plotting import plot_decision_regions from mlxtend.classifier import Adaline import matplotlib.pyplot as plt # Loading Data X, y = iris_data() X = X[:, [0, 3]] # sepal length and petal width X = X[0:100] # class 0 and class 1 y = y[0:100] # class 0 and class 1 # standardize X[:,0] = (X[:,0] - X[:,0].mean()) / X[:,0].std() X[:,1] = (X[:,1] - X[:,1].mean()) / X[:,1].std() ada = Adaline(epochs=15, eta=0.02, minibatches=len(y), # for SGD learning random_seed=1, print_progress=3) ada.fit(X, y) plot_decision_regions(X, y, clf=ada) plt.title('Adaline - Stochastic Gradient Descent') plt.show() plt.plot(range(len(ada.cost_)), ada.cost_) plt.xlabel('Iterations') plt.ylabel('Cost') plt.show() Iteration: 15/15 | Cost 3.81 | Elapsed: 0:00:00 | ETA: 0:00:00 Example 4 - Stochastic Gradient Descent with Minibatches from mlxtend.data import iris_data from mlxtend.plotting import plot_decision_regions from mlxtend.classifier import Adaline import matplotlib.pyplot as plt # Loading Data X, y = iris_data() X = X[:, [0, 3]] # sepal length and petal width X = X[0:100] # class 0 and class 1 y = y[0:100] # class 0 and class 1 # standardize X[:,0] = (X[:,0] - X[:,0].mean()) / X[:,0].std() X[:,1] = (X[:,1] - X[:,1].mean()) / X[:,1].std() ada = Adaline(epochs=15, eta=0.02, minibatches=5, # for SGD learning w. minibatch size 20 random_seed=1, print_progress=3) ada.fit(X, y) plot_decision_regions(X, y, clf=ada) plt.title('Adaline - Stochastic Gradient Descent w. Minibatches') plt.show() plt.plot(range(len(ada.cost_)), ada.cost_) plt.xlabel('Iterations') plt.ylabel('Cost') plt.show() Iteration: 15/15 | Cost 3.87 | Elapsed: 0:00:00 | ETA: 0:00:00 API Adaline(eta=0.01, epochs=50, minibatches=None, random_seed=None, print_progress=0) ADAptive LInear NEuron classifier. Note that this implementation of Adaline expects binary class labels in {0, 1}. Parameters eta : float (default: 0.01) solver rate (between 0.0 and 1.0) epochs : int (default: 50) Passes over the training dataset. Prior to each epoch, the dataset is shuffled if minibatches > 1 to prevent cycles in stochastic gradient descent. minibatches : int (default: None) The number of minibatches for gradient-based optimization. If None: Normal Equations (closed-form solution) If 1: Gradient Descent learning If len(y): Stochastic Gradient Descent (SGD) online learning If 1 < minibatches < len(y): SGD Minibatch learning random_seed : int (default: None) Set random state for shuffling and initializing the weights. print_progress : int (default: 0) Prints progress in fitting to stderr if not solver='normal equation' 0: No output 1: Epochs elapsed and cost 2: 1 plus time elapsed 3: 2 plus estimated time until completion Attributes w_ : 2d-array, shape={n_features, 1} Model weights after fitting. b_ : 1d-array, shape={1,} Bias unit after fitting. cost_ : list Sum of squared errors after each epoch. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/classifier/Adaline/ Methods fit(X, y, init_params=True) Learn model from training data. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values. init_params : bool (default: True) Re-initializes model parameters prior to fitting. Set False to continue training with weights from a previous model fitting. Returns self : object get_params(deep=True) Get parameters for this estimator. Parameters deep : boolean, optional If True, will return the parameters for this estimator and contained subobjects that are estimators. Returns params : mapping of string to any Parameter names mapped to their values.' adapted from https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/base.py Author: Gael Varoquaux gael.varoquaux@normalesup.org License: BSD 3 clause predict(X) Predict targets from X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns target_values : array-like, shape = [n_samples] Predicted target values. score(X, y) Compute the prediction accuracy Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values (true class labels). Returns acc : float The prediction accuracy as a float between 0.0 and 1.0 (perfect score). set_params( params) Set the parameters of this estimator. The method works on simple estimators as well as on nested objects (such as pipelines). The latter have parameters of the form <component>__<parameter> so that it's possible to update each component of a nested object. Returns self adapted from https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/base.py Author: Gael Varoquaux gael.varoquaux@normalesup.org License: BSD 3 clause","title":"Adaline: Adaptive Linear Neuron Classifier"},{"location":"user_guide/classifier/Adaline/#adaline-adaptive-linear-neuron-classifier","text":"An implementation of the ADAptive LInear NEuron, Adaline, for binary classification tasks. from mlxtend.classifier import Adaline","title":"Adaline: Adaptive Linear Neuron Classifier"},{"location":"user_guide/classifier/Adaline/#overview","text":"An illustration of the ADAptive LInear NEuron (Adaline) -- a single-layer artificial linear neuron with a threshold unit: The Adaline classifier is closely related to the Ordinary Least Squares (OLS) Linear Regression algorithm; in OLS regression we find the line (or hyperplane) that minimizes the vertical offsets. Or in other words, we define the best-fitting line as the line that minimizes the sum of squared errors (SSE) or mean squared error (MSE) between our target variable (y) and our predicted output over all samples i in our dataset of size n . SSE = \\sum_i (\\text{target}^{(i)} - \\text{output}^{(i)})^2 MSE = \\frac{1}{n} \\times SSE LinearRegression implements a linear regression model for performing ordinary least squares regression, and in Adaline, we add a threshold function g(\\cdot) to convert the continuous outcome to a categorical class label: $$y = g({z}) = \\begin{cases} 1 & \\text{if z $\\ge$ 0}\\\\ -1 & \\text{otherwise}. \\end{cases} $$ An Adaline model can be trained by one of the following three approaches: Normal Equations Gradient Descent Stochastic Gradient Descent","title":"Overview"},{"location":"user_guide/classifier/Adaline/#normal-equations-closed-form-solution","text":"The closed-form solution should be preferred for \"smaller\" datasets where calculating (a \"costly\") matrix inverse is not a concern. For very large datasets, or datasets where the inverse of [X^T X] may not exist (the matrix is non-invertible or singular, e.g., in case of perfect multicollinearity), the gradient descent or stochastic gradient descent approaches are to be preferred. The linear function (linear regression model) is defined as: z = w_0x_0 + w_1x_1 + ... + w_mx_m = \\sum_{j=0}^{m} w_j x_j = \\mathbf{w}^T\\mathbf{x} where y is the response variable, \\mathbf{x} is an m -dimensional sample vector, and \\mathbf{w} is the weight vector (vector of coefficients). Note that w_0 represents the y-axis intercept of the model and therefore x_0=1 . Using the closed-form solution (normal equation), we compute the weights of the model as follows: \\mathbf{w} = (\\mathbf{X}^T\\mathbf{X})^{-1}\\mathbf{X}^Ty","title":"Normal Equations (closed-form solution)"},{"location":"user_guide/classifier/Adaline/#gradient-descent-gd-and-stochastic-gradient-descent-sgd","text":"In the current implementation, the Adaline model is learned via Gradient Descent or Stochastic Gradient Descent. See Gradient Descent and Stochastic Gradient Descent and Deriving the Gradient Descent Rule for Linear Regression and Adaline for details. Random shuffling is implemented as: for one or more epochs randomly shuffle samples in the training set for training sample i compute gradients and perform weight updates","title":"Gradient Descent (GD)  and Stochastic Gradient Descent (SGD)"},{"location":"user_guide/classifier/Adaline/#references","text":"B. Widrow, M. E. Hoff, et al. Adaptive switching circuits . 1960.","title":"References"},{"location":"user_guide/classifier/Adaline/#example-1-closed-form-solution","text":"from mlxtend.data import iris_data from mlxtend.plotting import plot_decision_regions from mlxtend.classifier import Adaline import matplotlib.pyplot as plt # Loading Data X, y = iris_data() X = X[:, [0, 3]] # sepal length and petal width X = X[0:100] # class 0 and class 1 y = y[0:100] # class 0 and class 1 # standardize X[:,0] = (X[:,0] - X[:,0].mean()) / X[:,0].std() X[:,1] = (X[:,1] - X[:,1].mean()) / X[:,1].std() ada = Adaline(epochs=30, eta=0.01, minibatches=None, random_seed=1) ada.fit(X, y) plot_decision_regions(X, y, clf=ada) plt.title('Adaline - Closed Form') plt.show()","title":"Example 1 - Closed Form Solution"},{"location":"user_guide/classifier/Adaline/#example-2-gradient-descent","text":"from mlxtend.data import iris_data from mlxtend.plotting import plot_decision_regions from mlxtend.classifier import Adaline import matplotlib.pyplot as plt # Loading Data X, y = iris_data() X = X[:, [0, 3]] # sepal length and petal width X = X[0:100] # class 0 and class 1 y = y[0:100] # class 0 and class 1 # standardize X[:,0] = (X[:,0] - X[:,0].mean()) / X[:,0].std() X[:,1] = (X[:,1] - X[:,1].mean()) / X[:,1].std() ada = Adaline(epochs=30, eta=0.01, minibatches=1, # for Gradient Descent Learning random_seed=1, print_progress=3) ada.fit(X, y) plot_decision_regions(X, y, clf=ada) plt.title('Adaline - Gradient Descent') plt.show() plt.plot(range(len(ada.cost_)), ada.cost_) plt.xlabel('Iterations') plt.ylabel('Cost') Iteration: 30/30 | Cost 3.79 | Elapsed: 0:00:00 | ETA: 0:00:00 Text(0, 0.5, 'Cost')","title":"Example 2 - Gradient Descent"},{"location":"user_guide/classifier/Adaline/#example-3-stochastic-gradient-descent","text":"from mlxtend.data import iris_data from mlxtend.plotting import plot_decision_regions from mlxtend.classifier import Adaline import matplotlib.pyplot as plt # Loading Data X, y = iris_data() X = X[:, [0, 3]] # sepal length and petal width X = X[0:100] # class 0 and class 1 y = y[0:100] # class 0 and class 1 # standardize X[:,0] = (X[:,0] - X[:,0].mean()) / X[:,0].std() X[:,1] = (X[:,1] - X[:,1].mean()) / X[:,1].std() ada = Adaline(epochs=15, eta=0.02, minibatches=len(y), # for SGD learning random_seed=1, print_progress=3) ada.fit(X, y) plot_decision_regions(X, y, clf=ada) plt.title('Adaline - Stochastic Gradient Descent') plt.show() plt.plot(range(len(ada.cost_)), ada.cost_) plt.xlabel('Iterations') plt.ylabel('Cost') plt.show() Iteration: 15/15 | Cost 3.81 | Elapsed: 0:00:00 | ETA: 0:00:00","title":"Example 3 - Stochastic Gradient Descent"},{"location":"user_guide/classifier/Adaline/#example-4-stochastic-gradient-descent-with-minibatches","text":"from mlxtend.data import iris_data from mlxtend.plotting import plot_decision_regions from mlxtend.classifier import Adaline import matplotlib.pyplot as plt # Loading Data X, y = iris_data() X = X[:, [0, 3]] # sepal length and petal width X = X[0:100] # class 0 and class 1 y = y[0:100] # class 0 and class 1 # standardize X[:,0] = (X[:,0] - X[:,0].mean()) / X[:,0].std() X[:,1] = (X[:,1] - X[:,1].mean()) / X[:,1].std() ada = Adaline(epochs=15, eta=0.02, minibatches=5, # for SGD learning w. minibatch size 20 random_seed=1, print_progress=3) ada.fit(X, y) plot_decision_regions(X, y, clf=ada) plt.title('Adaline - Stochastic Gradient Descent w. Minibatches') plt.show() plt.plot(range(len(ada.cost_)), ada.cost_) plt.xlabel('Iterations') plt.ylabel('Cost') plt.show() Iteration: 15/15 | Cost 3.87 | Elapsed: 0:00:00 | ETA: 0:00:00","title":"Example 4 - Stochastic Gradient Descent with Minibatches"},{"location":"user_guide/classifier/Adaline/#api","text":"Adaline(eta=0.01, epochs=50, minibatches=None, random_seed=None, print_progress=0) ADAptive LInear NEuron classifier. Note that this implementation of Adaline expects binary class labels in {0, 1}. Parameters eta : float (default: 0.01) solver rate (between 0.0 and 1.0) epochs : int (default: 50) Passes over the training dataset. Prior to each epoch, the dataset is shuffled if minibatches > 1 to prevent cycles in stochastic gradient descent. minibatches : int (default: None) The number of minibatches for gradient-based optimization. If None: Normal Equations (closed-form solution) If 1: Gradient Descent learning If len(y): Stochastic Gradient Descent (SGD) online learning If 1 < minibatches < len(y): SGD Minibatch learning random_seed : int (default: None) Set random state for shuffling and initializing the weights. print_progress : int (default: 0) Prints progress in fitting to stderr if not solver='normal equation' 0: No output 1: Epochs elapsed and cost 2: 1 plus time elapsed 3: 2 plus estimated time until completion Attributes w_ : 2d-array, shape={n_features, 1} Model weights after fitting. b_ : 1d-array, shape={1,} Bias unit after fitting. cost_ : list Sum of squared errors after each epoch. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/classifier/Adaline/","title":"API"},{"location":"user_guide/classifier/Adaline/#methods","text":"fit(X, y, init_params=True) Learn model from training data. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values. init_params : bool (default: True) Re-initializes model parameters prior to fitting. Set False to continue training with weights from a previous model fitting. Returns self : object get_params(deep=True) Get parameters for this estimator. Parameters deep : boolean, optional If True, will return the parameters for this estimator and contained subobjects that are estimators. Returns params : mapping of string to any Parameter names mapped to their values.' adapted from https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/base.py Author: Gael Varoquaux gael.varoquaux@normalesup.org License: BSD 3 clause predict(X) Predict targets from X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns target_values : array-like, shape = [n_samples] Predicted target values. score(X, y) Compute the prediction accuracy Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values (true class labels). Returns acc : float The prediction accuracy as a float between 0.0 and 1.0 (perfect score). set_params( params) Set the parameters of this estimator. The method works on simple estimators as well as on nested objects (such as pipelines). The latter have parameters of the form <component>__<parameter> so that it's possible to update each component of a nested object. Returns self adapted from https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/base.py Author: Gael Varoquaux gael.varoquaux@normalesup.org License: BSD 3 clause","title":"Methods"},{"location":"user_guide/classifier/EnsembleVoteClassifier/","text":"EnsembleVoteClassifier: A majority voting classifier Implementation of a majority voting EnsembleVoteClassifier for classification. from mlxtend.classifier import EnsembleVoteClassifier Overview The EnsembleVoteClassifier is a meta-classifier for combining similar or conceptually different machine learning classifiers for classification via majority or plurality voting. (For simplicity, we will refer to both majority and plurality voting as majority voting.) The EnsembleVoteClassifier implements \"hard\" and \"soft\" voting. In hard voting, we predict the final class label as the class label that has been predicted most frequently by the classification models. In soft voting, we predict the class labels by averaging the class-probabilities (only recommended if the classifiers are well-calibrated). Note If you are interested in using the EnsembleVoteClassifier , please note that it is now also available through scikit learn (>0.17) as VotingClassifier . Majority Voting / Hard Voting Hard voting is the simplest case of majority voting. Here, we predict the class label \\hat{y} via majority (plurality) voting of each classifier C_j : \\hat{y}=mode\\{C_1(\\mathbf{x}), C_2(\\mathbf{x}), ..., C_m(\\mathbf{x})\\} Assuming that we combine three classifiers that classify a training sample as follows: classifier 1 -> class 0 classifier 2 -> class 0 classifier 3 -> class 1 \\hat{y}=mode\\{0, 0, 1\\} = 0 Via majority vote, we would classify the sample as \"class 0.\" Weighted Majority Vote In addition to the simple majority vote (hard voting) as described in the previous section, we can compute a weighted majority vote by associating a weight w_j with classifier C_j : \\hat{y} = \\arg \\max_i \\sum^{m}_{j=1} w_j \\chi_A \\big(C_j(\\mathbf{x})=i\\big), where \\chi_A is the characteristic function [C_j(\\mathbf{x}) = i \\; \\in A] , and A is the set of unique class labels. Continuing with the example from the previous section classifier 1 -> class 0 classifier 2 -> class 0 classifier 3 -> class 1 assigning the weights {0.2, 0.2, 0.6} would yield a prediction \\hat{y} = 1 : \\arg \\max_i [0.2 \\times i_0 + 0.2 \\times i_0 + 0.6 \\times i_1] = 1 Soft Voting In soft voting, we predict the class labels based on the predicted probabilities p for classifier -- this approach is only recommended if the classifiers are well-calibrated. \\hat{y} = \\arg \\max_i \\sum^{m}_{j=1} w_j p_{ij}, where w_j is the weight that can be assigned to the j th classifier. Assuming the example in the previous section was a binary classification task with class labels i \\in \\{0, 1\\} , our ensemble could make the following prediction: C_1(\\mathbf{x}) \\rightarrow [0.9, 0.1] C_2(\\mathbf{x}) \\rightarrow [0.8, 0.2] C_3(\\mathbf{x}) \\rightarrow [0.4, 0.6] Using uniform weights, we compute the average probabilities: p(i_0 \\mid \\mathbf{x}) = \\frac{0.9 + 0.8 + 0.4}{3} = 0.7 \\\\\\\\ p(i_1 \\mid \\mathbf{x}) = \\frac{0.1 + 0.2 + 0.6}{3} = 0.3 \\hat{y} = \\arg \\max_i \\big[p(i_0 \\mid \\mathbf{x}), p(i_1 \\mid \\mathbf{x}) \\big] = 0 However, assigning the weights {0.1, 0.1, 0.8} would yield a prediction \\hat{y} = 1 : p(i_0 \\mid \\mathbf{x}) = {0.1 \\times 0.9 + 0.1 \\times 0.8 + 0.8 \\times 0.4} = 0.49 \\\\\\\\ p(i_1 \\mid \\mathbf{x}) = {0.1 \\times 0.1 + 0.2 \\times 0.1 + 0.8 \\times 0.6} = 0.51 \\hat{y} = \\arg \\max_i \\big[p(i_0 \\mid \\mathbf{x}), p(i_1 \\mid \\mathbf{x}) \\big] = 1 References [1] S. Raschka. Python Machine Learning . Packt Publishing Ltd., 2015. Example 1 - Classifying Iris Flowers Using Different Classification Models from sklearn import datasets iris = datasets.load_iris() X, y = iris.data[:, 1:3], iris.target from sklearn import model_selection from sklearn.linear_model import LogisticRegression from sklearn.naive_bayes import GaussianNB from sklearn.ensemble import RandomForestClassifier import numpy as np clf1 = LogisticRegression(random_state=1) clf2 = RandomForestClassifier(random_state=1) clf3 = GaussianNB() print('5-fold cross validation:\\n') labels = ['Logistic Regression', 'Random Forest', 'Naive Bayes'] for clf, label in zip([clf1, clf2, clf3], labels): scores = model_selection.cross_val_score(clf, X, y, cv=5, scoring='accuracy') print(\"Accuracy: %0.2f (+/- %0.2f) [%s]\" % (scores.mean(), scores.std(), label)) 5-fold cross validation: Accuracy: 0.95 (+/- 0.04) [Logistic Regression] Accuracy: 0.94 (+/- 0.04) [Random Forest] Accuracy: 0.91 (+/- 0.04) [Naive Bayes] from mlxtend.classifier import EnsembleVoteClassifier eclf = EnsembleVoteClassifier(clfs=[clf1, clf2, clf3], weights=[1,1,1]) labels = ['Logistic Regression', 'Random Forest', 'Naive Bayes', 'Ensemble'] for clf, label in zip([clf1, clf2, clf3, eclf], labels): scores = model_selection.cross_val_score(clf, X, y, cv=5, scoring='accuracy') print(\"Accuracy: %0.2f (+/- %0.2f) [%s]\" % (scores.mean(), scores.std(), label)) Accuracy: 0.95 (+/- 0.04) [Logistic Regression] Accuracy: 0.94 (+/- 0.04) [Random Forest] Accuracy: 0.91 (+/- 0.04) [Naive Bayes] Accuracy: 0.95 (+/- 0.04) [Ensemble] Plotting Decision Regions import matplotlib.pyplot as plt from mlxtend.plotting import plot_decision_regions import matplotlib.gridspec as gridspec import itertools gs = gridspec.GridSpec(2, 2) fig = plt.figure(figsize=(10,8)) labels = ['Logistic Regression', 'Random Forest', 'Naive Bayes', 'Ensemble'] for clf, lab, grd in zip([clf1, clf2, clf3, eclf], labels, itertools.product([0, 1], repeat=2)): clf.fit(X, y) ax = plt.subplot(gs[grd[0], grd[1]]) fig = plot_decision_regions(X=X, y=y, clf=clf) plt.title(lab) Example 2 - Grid Search from sklearn import datasets iris = datasets.load_iris() X, y = iris.data[:, 1:3], iris.target %%capture --no-display from sklearn.model_selection import GridSearchCV from sklearn.linear_model import LogisticRegression from sklearn.naive_bayes import GaussianNB from sklearn.ensemble import RandomForestClassifier from mlxtend.classifier import EnsembleVoteClassifier clf1 = LogisticRegression(random_state=1) clf2 = RandomForestClassifier(random_state=1) clf3 = GaussianNB() eclf = EnsembleVoteClassifier(clfs=[clf1, clf2, clf3], voting='soft') params = {'logisticregression__C': [1.0, 100.0], 'randomforestclassifier__n_estimators': [20, 200],} grid = GridSearchCV(estimator=eclf, param_grid=params, cv=5) grid.fit(iris.data, iris.target) cv_keys = ('mean_test_score', 'std_test_score', 'params') for r, _ in enumerate(grid.cv_results_['mean_test_score']): print(\"%0.3f +/- %0.2f %r\" % (grid.cv_results_[cv_keys[0]][r], grid.cv_results_[cv_keys[1]][r] / 2.0, grid.cv_results_[cv_keys[2]][r])) Note : If the EnsembleClassifier is initialized with multiple similar estimator objects, the estimator names are modified with consecutive integer indices, for example: %%capture --no-display clf1 = LogisticRegression(random_state=1) clf2 = RandomForestClassifier(random_state=1) eclf = EnsembleVoteClassifier(clfs=[clf1, clf1, clf2], voting='soft') params = {'logisticregression-1__C': [1.0, 100.0], 'logisticregression-2__C': [1.0, 100.0], 'randomforestclassifier__n_estimators': [20, 200],} grid = GridSearchCV(estimator=eclf, param_grid=params, cv=5) grid = grid.fit(iris.data, iris.target) Note The EnsembleVoteClass also enables grid search over the clfs argument. However, due to the current implementation of GridSearchCV in scikit-learn, it is not possible to search over both, differenct classifiers and classifier parameters at the same time. For instance, while the following parameter dictionary works params = {'randomforestclassifier__n_estimators': [1, 100], 'clfs': [(clf1, clf1, clf1), (clf2, clf3)]} it will use the instance settings of clf1 , clf2 , and clf3 and not overwrite it with the 'n_estimators' settings from 'randomforestclassifier__n_estimators': [1, 100] . Example 3 - Majority voting with classifiers trained on different feature subsets Feature selection algorithms implemented in scikit-learn as well as the SequentialFeatureSelector implement a transform method that passes the reduced feature subset to the next item in a Pipeline . For example, the method def transform(self, X): return X[:, self.k_feature_idx_] returns the best feature columns, k_feature_idx_ , given a dataset X. Thus, we simply need to construct a Pipeline consisting of the feature selector and the classifier in order to select different feature subsets for different algorithms. During fitting , the optimal feature subsets are automatically determined via the GridSearchCV object, and by calling predict , the fitted feature selector in the pipeline only passes these columns along, which resulted in the best performance for the respective classifier. %%capture --no-display from sklearn import datasets iris = datasets.load_iris() X, y = iris.data[:, :], iris.target from sklearn.model_selection import GridSearchCV from sklearn.linear_model import LogisticRegression from sklearn.naive_bayes import GaussianNB from sklearn.ensemble import RandomForestClassifier from mlxtend.classifier import EnsembleVoteClassifier from sklearn.pipeline import Pipeline from mlxtend.feature_selection import SequentialFeatureSelector clf1 = LogisticRegression(random_state=1) clf2 = RandomForestClassifier(random_state=1) clf3 = GaussianNB() # Creating a feature-selection-classifier pipeline sfs1 = SequentialFeatureSelector(clf1, k_features=4, forward=True, floating=False, scoring='accuracy', verbose=0, cv=0) clf1_pipe = Pipeline([('sfs', sfs1), ('logreg', clf1)]) eclf = EnsembleVoteClassifier(clfs=[clf1_pipe, clf2, clf3], voting='soft') params = {'pipeline__sfs__k_features': [1, 2, 3], 'pipeline__logreg__C': [1.0, 100.0], 'randomforestclassifier__n_estimators': [20, 200]} grid = GridSearchCV(estimator=eclf, param_grid=params, cv=5) grid.fit(iris.data, iris.target) cv_keys = ('mean_test_score', 'std_test_score', 'params') for r, _ in enumerate(grid.cv_results_['mean_test_score']): print(\"%0.3f +/- %0.2f %r\" % (grid.cv_results_[cv_keys[0]][r], grid.cv_results_[cv_keys[1]][r] / 2.0, grid.cv_results_[cv_keys[2]][r])) The best parameters determined via GridSearch are: grid.best_params_ {'pipeline__logreg__C': 1.0, 'pipeline__sfs__k_features': 2, 'randomforestclassifier__n_estimators': 200} Now, we assign these parameters to the ensemble voting classifier, fit the models on the complete training set, and perform a prediction on 3 samples from the Iris dataset. eclf = eclf.set_params(**grid.best_params_) eclf.fit(X, y).predict(X[[1, 51, 149]]) array([0, 1, 2]) Manual Approach Alternatively, we can select different columns \"manually\" using the ColumnSelector object. In this example, we select only the first (sepal length) and third (petal length) column for the logistic regression classifier ( clf1 ). from mlxtend.feature_selection import ColumnSelector col_sel = ColumnSelector(cols=[0, 2]) clf1_pipe = Pipeline([('sel', col_sel), ('logreg', clf1)]) eclf = EnsembleVoteClassifier(clfs=[clf1_pipe, clf2, clf3], voting='soft') eclf.fit(X, y).predict(X[[1, 51, 149]]) array([0, 1, 2]) Furthermore, we can fit the SequentialFeatureSelector separately, outside the grid search hyperparameter optimization pipeline. Here, we determine the best features first, and then we construct a pipeline using these \"fixed,\" best features as seed for the ColumnSelector : sfs1 = SequentialFeatureSelector(clf1, k_features=2, forward=True, floating=False, scoring='accuracy', verbose=1, cv=0) sfs1.fit(X, y) print('Best features', sfs1.k_feature_idx_) col_sel = ColumnSelector(cols=sfs1.k_feature_idx_) clf1_pipe = Pipeline([('sel', col_sel), ('logreg', clf1)]) [Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers. [Parallel(n_jobs=1)]: Done 4 out of 4 | elapsed: 0.0s finished Features: 1/2[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers. [Parallel(n_jobs=1)]: Done 3 out of 3 | elapsed: 0.0s finished Features: 2/2 Best features (2, 3) eclf = EnsembleVoteClassifier(clfs=[clf1_pipe, clf2, clf3], voting='soft') eclf.fit(X, y).predict(X[[1, 51, 149]]) array([0, 1, 2]) Example 5 - Using Pre-fitted Classifiers from sklearn import datasets iris = datasets.load_iris() X, y = iris.data[:, 1:3], iris.target Assume that we previously fitted our classifiers: from sklearn import model_selection from sklearn.linear_model import LogisticRegression from sklearn.naive_bayes import GaussianNB from sklearn.ensemble import RandomForestClassifier import numpy as np clf1 = LogisticRegression(random_state=1) clf2 = RandomForestClassifier(random_state=1) clf3 = GaussianNB() for clf in (clf1, clf2, clf3): clf.fit(X, y) By setting fit_base_estimators=False , it will enforce use_clones to be False and the EnsembleVoteClassifier will not re-fit these classifers to save computational time: from mlxtend.classifier import EnsembleVoteClassifier import copy eclf = EnsembleVoteClassifier(clfs=[clf1, clf2, clf3], weights=[1,1,1], fit_base_estimators=False) labels = ['Logistic Regression', 'Random Forest', 'Naive Bayes', 'Ensemble'] eclf.fit(X, y) print('accuracy:', np.mean(y == eclf.predict(X))) accuracy: 0.96 /Users/sebastian/miniforge3/lib/python3.9/site-packages/mlxtend/classifier/ensemble_vote.py:166: UserWarning: fit_base_estimators=False enforces use_clones to be `False` warnings.warn(\"fit_base_estimators=False \" However, please note that fit_base_estimators=False is incompatible to any form of cross-validation that is done in e.g., model_selection.cross_val_score or model_selection.GridSearchCV , etc., since it would require the classifiers to be refit to the training folds. Thus, only use fit_base_estimators=False if you want to make a prediction directly without cross-validation. Example 6 - Ensembles of Classifiers that Operate on Different Feature Subsets If desired, the different classifiers can be fit to different subsets of features in the training dataset. The following example illustrates how this can be done on a technical level using scikit-learn pipelines and the ColumnSelector : from sklearn.datasets import load_iris from mlxtend.classifier import EnsembleVoteClassifier from mlxtend.feature_selection import ColumnSelector from sklearn.pipeline import make_pipeline from sklearn.linear_model import LogisticRegression iris = load_iris() X = iris.data y = iris.target pipe1 = make_pipeline(ColumnSelector(cols=(0, 2)), LogisticRegression()) pipe2 = make_pipeline(ColumnSelector(cols=(1, 2, 3)), LogisticRegression()) eclf = EnsembleVoteClassifier(clfs=[pipe1, pipe2]) eclf.fit(X, y) EnsembleVoteClassifier(clfs=[Pipeline(steps=[('columnselector', ColumnSelector(cols=(0, 2))), ('logisticregression', LogisticRegression())]), Pipeline(steps=[('columnselector', ColumnSelector(cols=(1, 2, 3))), ('logisticregression', LogisticRegression())])]) Example 7 - A Note about Scikit-Learn SVMs and Soft Voting This section provides some additional technical insights in how probabilities are used when voting='soft' . Note that scikit-learn estimates the probabilities for SVMs (more info here: https://scikit-learn.org/stable/modules/svm.html#scores-probabilities) in a way that these may not be consistent with the class labels that the SVM predicts. This is an extreme example, but let's say we have a dataset with 3 class labels, 0, 1, and 2. For a given training example, the SVM classifier may predict class 2. However, the class-membership probabilities may look as follows: class 0: 99% class 1: 0.5% class 2: 0.5% A practical example of this scenario is shown below: import numpy as np from mlxtend.classifier import EnsembleVoteClassifier from sklearn.svm import SVC from sklearn.datasets import load_iris iris = load_iris() X, y = iris.data, iris.target clf2 = SVC(probability=True, random_state=4) clf2.fit(X, y) eclf = EnsembleVoteClassifier(clfs=[clf2], voting='soft', fit_base_estimators=False) eclf.fit(X, y) for svm_class, e_class, svm_prob, e_prob, in zip(clf2.predict(X), eclf.predict(X), clf2.predict_proba(X), eclf.predict_proba(X)): if svm_class != e_class: print('============') print('Probas from SVM :', svm_prob) print('Class from SVM :', svm_class) print('Probas from SVM in Ensemble:', e_prob) print('Class from SVM in Ensemble :', e_class) print('============') ============ Probas from SVM : [0.00921708 0.49415165 0.49663127] Class from SVM : 1 Probas from SVM in Ensemble: [0.00921708 0.49415165 0.49663127] Class from SVM in Ensemble : 2 ============ /Users/sebastian/miniforge3/lib/python3.9/site-packages/mlxtend/classifier/ensemble_vote.py:166: UserWarning: fit_base_estimators=False enforces use_clones to be `False` warnings.warn(\"fit_base_estimators=False \" Example 7 - A Note about Scikit-Learn SVMs and Soft Voting Based on the probabilities, we would expect the SVM to predict class 2, because it has the highest probability. Since the EnsembleVoteClassifier uses the argmax function internally if voting='soft' , it would indeed predict class 2 in this case even if the ensemble consists of only one SVM model. Note that in practice, this minor technical detail does not need to concern you, but it is useful to keep it in mind in case you are wondering about results from a 1-model SVM ensemble compared to that SVM alone -- this is not a bug. Example 8 - Optimizing Ensemble Weights with Nelder-Mead In this section, we will see how we can use a heuristic search method like Nelder-Mead for optimizing the ensemble weights. Suppose we have the following example scenario where we fit 3 individual classifiers on different subsets of the training dataset: from mlxtend.classifier import EnsembleVoteClassifier from mlxtend.data import mnist_data from sklearn.linear_model import LogisticRegression from sklearn.model_selection import train_test_split from sklearn.naive_bayes import GaussianNB from sklearn.tree import DecisionTreeClassifier X, y = mnist_data() X_train, X_val, y_train, y_val = train_test_split( X, y, test_size=0.5, shuffle=True, random_state=1 ) clf1 = GaussianNB() clf2 = LogisticRegression(random_state=123, solver='newton-cg') clf3 = DecisionTreeClassifier(random_state=123, max_depth=2) clf1.fit(X_train[500:1000], y_train[500:1000]) clf2.fit(X_train[750:1250], y_train[750:1250]) clf3.fit(X_train[1250:2000], y_train[1250:2000]); Then, we construct the an ensemble classifier from these 3 classifiers where each classifier contributes equally with weight 1: eclf = EnsembleVoteClassifier( clfs=(clf1, clf2, clf3), voting=\"soft\", # the same would also work with \"hard\" voting weights=(1, 1, 1), use_clones=False, fit_base_estimators=False, ) eclf.fit(X_train, y_train) eclf.score(X_val, y_val) /Users/sebastian/miniforge3/lib/python3.9/site-packages/mlxtend/classifier/ensemble_vote.py:166: UserWarning: fit_base_estimators=False enforces use_clones to be `False` warnings.warn(\"fit_base_estimators=False \" 0.8012 We see that we reach 80% accuracy on the validation set. Can we do better? Maybe they indvidual classifiers shouldn't be contributing equally. Perhaps, we can use an optimization algorothm from scipy.optimize to find a better relative weighting of these individual classifiers. Let's set up an objective function that we want to minimize via SciPy's minimize : from scipy.optimize import minimize def function_to_minimize(weights, fitted_clfs): w1, w2 = weights # these are the new weights! newclf = EnsembleVoteClassifier( voting=\"soft\", use_clones=False, fit_base_estimators=False, clfs=fitted_clfs, weights=(w1, w2, 1.), # use the new weights ) newclf.fit(X_train, y_train) score = newclf.score(X_val, y_val) # change accuracy to error so that smaller is better score_to_minimize = 1 - score return score_to_minimize Note a few things: We only optimize 2 out of the 3 classifier weigths. That's because the weighting is relative to each other, and it would be overkill (and too many degrees of freedom) to also optimize weight 3. We set use_clones=False & fit_base_estimators=False as before, this is to make sure that we use the prefit classifiers in the ensemble classifier. Instead of optimizing the accuracy, we optimize the the classification error, score_to_minimize = 1 - score . That's because we use the minimize function where lower means better. Next, let's choose some initial weight values and run the optimization. Via the bounds we specify the range (lower and upper value) for each weight so that the search doesn't go crazy: %%capture --no-display init_weights = [1., 1.] results = minimize( function_to_minimize, init_weights, args=((clf1, clf2, clf3),), bounds=((0, 5), (0, 5)), method=\"nelder-mead\", ) Let's look at the results! print(results) final_simplex: (array([[0.575 , 1.40625 ], [0.57500153, 1.40622215], [0.57508965, 1.40617647]]), array([0.1324, 0.1324, 0.1324])) fun: 0.13239999999999996 message: 'Optimization terminated successfully.' nfev: 60 nit: 21 status: 0 success: True x: array([0.575 , 1.40625]) It looks like the search was successful and returned the following weights: solution = results[\"x\"] print(solution) [0.575 1.40625] Let's use these new weights in our ensemble classifier: eclf = EnsembleVoteClassifier( clfs=(clf1, clf2, clf3), voting=\"soft\", weights=(solution[0], solution[1], 1), use_clones=False, fit_base_estimators=False, ) eclf.fit(X_train, y_train) eclf.score(X_val, y_val) /Users/sebastian/miniforge3/lib/python3.9/site-packages/mlxtend/classifier/ensemble_vote.py:166: UserWarning: fit_base_estimators=False enforces use_clones to be `False` warnings.warn(\"fit_base_estimators=False \" 0.8676 As we can see, the results on the validation set (0.8676) improved compared to the original ones (0.8012). Yay! API EnsembleVoteClassifier(clfs, voting='hard', weights=None, verbose=0, use_clones=True, fit_base_estimators=True) Soft Voting/Majority Rule classifier for scikit-learn estimators. Parameters clfs : array-like, shape = [n_classifiers] A list of classifiers. Invoking the fit method on the VotingClassifier will fit clones of those original classifiers be stored in the class attribute if use_clones=True (default) and fit_base_estimators=True (default). voting : str, {'hard', 'soft'} (default='hard') If 'hard', uses predicted class labels for majority rule voting. Else if 'soft', predicts the class label based on the argmax of the sums of the predicted probalities, which is recommended for an ensemble of well-calibrated classifiers. weights : array-like, shape = [n_classifiers], optional (default= None ) Sequence of weights ( float or int ) to weight the occurances of predicted class labels ( hard voting) or class probabilities before averaging ( soft voting). Uses uniform weights if None . verbose : int, optional (default=0) Controls the verbosity of the building process. - verbose=0 (default): Prints nothing - verbose=1 : Prints the number & name of the clf being fitted - verbose=2 : Prints info about the parameters of the clf being fitted - verbose>2 : Changes verbose param of the underlying clf to self.verbose - 2 use_clones : bool (default: True) Clones the classifiers for stacking classification if True (default) or else uses the original ones, which will be refitted on the dataset upon calling the fit method. Hence, if use_clones=True, the original input classifiers will remain unmodified upon using the StackingClassifier's fit method. Setting use_clones=False is recommended if you are working with estimators that are supporting the scikit-learn fit/predict API interface but are not compatible to scikit-learn's clone function. fit_base_estimators : bool (default: True) Refits classifiers in clfs if True; uses references to the clfs , otherwise (assumes that the classifiers were already fit). Note: fit_base_estimators=False will enforce use_clones to be False, and is incompatible to most scikit-learn wrappers! For instance, if any form of cross-validation is performed this would require the re-fitting classifiers to training folds, which would raise a NotFitterError if fit_base_estimators=False. (New in mlxtend v0.6.) Attributes classes_ : array-like, shape = [n_predictions] clf : array-like, shape = [n_predictions] The input classifiers; may be overwritten if use_clones=False clf_ : array-like, shape = [n_predictions] Fitted input classifiers; clones if use_clones=True Examples ``` >>> import numpy as np >>> from sklearn.linear_model import LogisticRegression >>> from sklearn.naive_bayes import GaussianNB >>> from sklearn.ensemble import RandomForestClassifier >>> from mlxtend.sklearn import EnsembleVoteClassifier >>> clf1 = LogisticRegression(random_seed=1) >>> clf2 = RandomForestClassifier(random_seed=1) >>> clf3 = GaussianNB() >>> X = np.array([[-1, -1], [-2, -1], [-3, -2], [1, 1], [2, 1], [3, 2]]) >>> y = np.array([1, 1, 1, 2, 2, 2]) >>> eclf1 = EnsembleVoteClassifier(clfs=[clf1, clf2, clf3], ... voting='hard', verbose=1) >>> eclf1 = eclf1.fit(X, y) >>> print(eclf1.predict(X)) [1 1 1 2 2 2] >>> eclf2 = EnsembleVoteClassifier(clfs=[clf1, clf2, clf3], voting='soft') >>> eclf2 = eclf2.fit(X, y) >>> print(eclf2.predict(X)) [1 1 1 2 2 2] >>> eclf3 = EnsembleVoteClassifier(clfs=[clf1, clf2, clf3], ... voting='soft', weights=[2,1,1]) >>> eclf3 = eclf3.fit(X, y) >>> print(eclf3.predict(X)) [1 1 1 2 2 2] >>> For more usage examples, please see https://rasbt.github.io/mlxtend/user_guide/classifier/EnsembleVoteClassifier/ ``` Methods fit(X, y, sample_weight=None) Learn weight coefficients from training data for each classifier. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values. sample_weight : array-like, shape = [n_samples], optional Sample weights passed as sample_weights to each regressor in the regressors list as well as the meta_regressor. Raises error if some regressor does not support sample_weight in the fit() method. Returns self : object fit_transform(X, y=None, fit_params) Fit to data, then transform it. Fits transformer to `X` and `y` with optional parameters `fit_params` and returns a transformed version of `X`. Parameters X : array-like of shape (n_samples, n_features) Input samples. y : array-like of shape (n_samples,) or (n_samples, n_outputs), default=None Target values (None for unsupervised transformations). **fit_params : dict Additional fit parameters. Returns X_new : ndarray array of shape (n_samples, n_features_new) Transformed array. get_params(deep=True) Return estimator parameter names for GridSearch support. predict(X) Predict class labels for X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns maj : array-like, shape = [n_samples] Predicted class labels. predict_proba(X) Predict class probabilities for X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns avg : array-like, shape = [n_samples, n_classes] Weighted average probability for each class per sample. score(X, y, sample_weight=None) Return the mean accuracy on the given test data and labels. In multi-label classification, this is the subset accuracy which is a harsh metric since you require for each sample that each label set be correctly predicted. Parameters X : array-like of shape (n_samples, n_features) Test samples. y : array-like of shape (n_samples,) or (n_samples, n_outputs) True labels for X . sample_weight : array-like of shape (n_samples,), default=None Sample weights. Returns score : float Mean accuracy of self.predict(X) wrt. y . set_params( params) Set the parameters of this estimator. The method works on simple estimators as well as on nested objects (such as :class:`~sklearn.pipeline.Pipeline`). The latter have parameters of the form ``<component>__<parameter>`` so that it's possible to update each component of a nested object. Parameters **params : dict Estimator parameters. Returns self : estimator instance Estimator instance. transform(X) Return class labels or probabilities for X for each estimator. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns If voting='soft'`` : array-like = [n_classifiers, n_samples, n_classes] Class probabilties calculated by each classifier. If voting='hard'`` : array-like = [n_classifiers, n_samples] Class labels predicted by each classifier. ython","title":"EnsembleVoteClassifier: A majority voting classifier"},{"location":"user_guide/classifier/EnsembleVoteClassifier/#ensemblevoteclassifier-a-majority-voting-classifier","text":"Implementation of a majority voting EnsembleVoteClassifier for classification. from mlxtend.classifier import EnsembleVoteClassifier","title":"EnsembleVoteClassifier: A majority voting classifier"},{"location":"user_guide/classifier/EnsembleVoteClassifier/#overview","text":"The EnsembleVoteClassifier is a meta-classifier for combining similar or conceptually different machine learning classifiers for classification via majority or plurality voting. (For simplicity, we will refer to both majority and plurality voting as majority voting.) The EnsembleVoteClassifier implements \"hard\" and \"soft\" voting. In hard voting, we predict the final class label as the class label that has been predicted most frequently by the classification models. In soft voting, we predict the class labels by averaging the class-probabilities (only recommended if the classifiers are well-calibrated). Note If you are interested in using the EnsembleVoteClassifier , please note that it is now also available through scikit learn (>0.17) as VotingClassifier .","title":"Overview"},{"location":"user_guide/classifier/EnsembleVoteClassifier/#majority-voting-hard-voting","text":"Hard voting is the simplest case of majority voting. Here, we predict the class label \\hat{y} via majority (plurality) voting of each classifier C_j : \\hat{y}=mode\\{C_1(\\mathbf{x}), C_2(\\mathbf{x}), ..., C_m(\\mathbf{x})\\} Assuming that we combine three classifiers that classify a training sample as follows: classifier 1 -> class 0 classifier 2 -> class 0 classifier 3 -> class 1 \\hat{y}=mode\\{0, 0, 1\\} = 0 Via majority vote, we would classify the sample as \"class 0.\"","title":"Majority Voting / Hard Voting"},{"location":"user_guide/classifier/EnsembleVoteClassifier/#weighted-majority-vote","text":"In addition to the simple majority vote (hard voting) as described in the previous section, we can compute a weighted majority vote by associating a weight w_j with classifier C_j : \\hat{y} = \\arg \\max_i \\sum^{m}_{j=1} w_j \\chi_A \\big(C_j(\\mathbf{x})=i\\big), where \\chi_A is the characteristic function [C_j(\\mathbf{x}) = i \\; \\in A] , and A is the set of unique class labels. Continuing with the example from the previous section classifier 1 -> class 0 classifier 2 -> class 0 classifier 3 -> class 1 assigning the weights {0.2, 0.2, 0.6} would yield a prediction \\hat{y} = 1 : \\arg \\max_i [0.2 \\times i_0 + 0.2 \\times i_0 + 0.6 \\times i_1] = 1","title":"Weighted Majority Vote"},{"location":"user_guide/classifier/EnsembleVoteClassifier/#soft-voting","text":"In soft voting, we predict the class labels based on the predicted probabilities p for classifier -- this approach is only recommended if the classifiers are well-calibrated. \\hat{y} = \\arg \\max_i \\sum^{m}_{j=1} w_j p_{ij}, where w_j is the weight that can be assigned to the j th classifier. Assuming the example in the previous section was a binary classification task with class labels i \\in \\{0, 1\\} , our ensemble could make the following prediction: C_1(\\mathbf{x}) \\rightarrow [0.9, 0.1] C_2(\\mathbf{x}) \\rightarrow [0.8, 0.2] C_3(\\mathbf{x}) \\rightarrow [0.4, 0.6] Using uniform weights, we compute the average probabilities: p(i_0 \\mid \\mathbf{x}) = \\frac{0.9 + 0.8 + 0.4}{3} = 0.7 \\\\\\\\ p(i_1 \\mid \\mathbf{x}) = \\frac{0.1 + 0.2 + 0.6}{3} = 0.3 \\hat{y} = \\arg \\max_i \\big[p(i_0 \\mid \\mathbf{x}), p(i_1 \\mid \\mathbf{x}) \\big] = 0 However, assigning the weights {0.1, 0.1, 0.8} would yield a prediction \\hat{y} = 1 : p(i_0 \\mid \\mathbf{x}) = {0.1 \\times 0.9 + 0.1 \\times 0.8 + 0.8 \\times 0.4} = 0.49 \\\\\\\\ p(i_1 \\mid \\mathbf{x}) = {0.1 \\times 0.1 + 0.2 \\times 0.1 + 0.8 \\times 0.6} = 0.51 \\hat{y} = \\arg \\max_i \\big[p(i_0 \\mid \\mathbf{x}), p(i_1 \\mid \\mathbf{x}) \\big] = 1","title":"Soft Voting"},{"location":"user_guide/classifier/EnsembleVoteClassifier/#references","text":"[1] S. Raschka. Python Machine Learning . Packt Publishing Ltd., 2015.","title":"References"},{"location":"user_guide/classifier/EnsembleVoteClassifier/#example-1-classifying-iris-flowers-using-different-classification-models","text":"from sklearn import datasets iris = datasets.load_iris() X, y = iris.data[:, 1:3], iris.target from sklearn import model_selection from sklearn.linear_model import LogisticRegression from sklearn.naive_bayes import GaussianNB from sklearn.ensemble import RandomForestClassifier import numpy as np clf1 = LogisticRegression(random_state=1) clf2 = RandomForestClassifier(random_state=1) clf3 = GaussianNB() print('5-fold cross validation:\\n') labels = ['Logistic Regression', 'Random Forest', 'Naive Bayes'] for clf, label in zip([clf1, clf2, clf3], labels): scores = model_selection.cross_val_score(clf, X, y, cv=5, scoring='accuracy') print(\"Accuracy: %0.2f (+/- %0.2f) [%s]\" % (scores.mean(), scores.std(), label)) 5-fold cross validation: Accuracy: 0.95 (+/- 0.04) [Logistic Regression] Accuracy: 0.94 (+/- 0.04) [Random Forest] Accuracy: 0.91 (+/- 0.04) [Naive Bayes] from mlxtend.classifier import EnsembleVoteClassifier eclf = EnsembleVoteClassifier(clfs=[clf1, clf2, clf3], weights=[1,1,1]) labels = ['Logistic Regression', 'Random Forest', 'Naive Bayes', 'Ensemble'] for clf, label in zip([clf1, clf2, clf3, eclf], labels): scores = model_selection.cross_val_score(clf, X, y, cv=5, scoring='accuracy') print(\"Accuracy: %0.2f (+/- %0.2f) [%s]\" % (scores.mean(), scores.std(), label)) Accuracy: 0.95 (+/- 0.04) [Logistic Regression] Accuracy: 0.94 (+/- 0.04) [Random Forest] Accuracy: 0.91 (+/- 0.04) [Naive Bayes] Accuracy: 0.95 (+/- 0.04) [Ensemble]","title":"Example 1 -  Classifying Iris Flowers Using Different Classification Models"},{"location":"user_guide/classifier/EnsembleVoteClassifier/#plotting-decision-regions","text":"import matplotlib.pyplot as plt from mlxtend.plotting import plot_decision_regions import matplotlib.gridspec as gridspec import itertools gs = gridspec.GridSpec(2, 2) fig = plt.figure(figsize=(10,8)) labels = ['Logistic Regression', 'Random Forest', 'Naive Bayes', 'Ensemble'] for clf, lab, grd in zip([clf1, clf2, clf3, eclf], labels, itertools.product([0, 1], repeat=2)): clf.fit(X, y) ax = plt.subplot(gs[grd[0], grd[1]]) fig = plot_decision_regions(X=X, y=y, clf=clf) plt.title(lab)","title":"Plotting Decision Regions"},{"location":"user_guide/classifier/EnsembleVoteClassifier/#example-2-grid-search","text":"from sklearn import datasets iris = datasets.load_iris() X, y = iris.data[:, 1:3], iris.target %%capture --no-display from sklearn.model_selection import GridSearchCV from sklearn.linear_model import LogisticRegression from sklearn.naive_bayes import GaussianNB from sklearn.ensemble import RandomForestClassifier from mlxtend.classifier import EnsembleVoteClassifier clf1 = LogisticRegression(random_state=1) clf2 = RandomForestClassifier(random_state=1) clf3 = GaussianNB() eclf = EnsembleVoteClassifier(clfs=[clf1, clf2, clf3], voting='soft') params = {'logisticregression__C': [1.0, 100.0], 'randomforestclassifier__n_estimators': [20, 200],} grid = GridSearchCV(estimator=eclf, param_grid=params, cv=5) grid.fit(iris.data, iris.target) cv_keys = ('mean_test_score', 'std_test_score', 'params') for r, _ in enumerate(grid.cv_results_['mean_test_score']): print(\"%0.3f +/- %0.2f %r\" % (grid.cv_results_[cv_keys[0]][r], grid.cv_results_[cv_keys[1]][r] / 2.0, grid.cv_results_[cv_keys[2]][r])) Note : If the EnsembleClassifier is initialized with multiple similar estimator objects, the estimator names are modified with consecutive integer indices, for example: %%capture --no-display clf1 = LogisticRegression(random_state=1) clf2 = RandomForestClassifier(random_state=1) eclf = EnsembleVoteClassifier(clfs=[clf1, clf1, clf2], voting='soft') params = {'logisticregression-1__C': [1.0, 100.0], 'logisticregression-2__C': [1.0, 100.0], 'randomforestclassifier__n_estimators': [20, 200],} grid = GridSearchCV(estimator=eclf, param_grid=params, cv=5) grid = grid.fit(iris.data, iris.target) Note The EnsembleVoteClass also enables grid search over the clfs argument. However, due to the current implementation of GridSearchCV in scikit-learn, it is not possible to search over both, differenct classifiers and classifier parameters at the same time. For instance, while the following parameter dictionary works params = {'randomforestclassifier__n_estimators': [1, 100], 'clfs': [(clf1, clf1, clf1), (clf2, clf3)]} it will use the instance settings of clf1 , clf2 , and clf3 and not overwrite it with the 'n_estimators' settings from 'randomforestclassifier__n_estimators': [1, 100] .","title":"Example 2 - Grid Search"},{"location":"user_guide/classifier/EnsembleVoteClassifier/#example-3-majority-voting-with-classifiers-trained-on-different-feature-subsets","text":"Feature selection algorithms implemented in scikit-learn as well as the SequentialFeatureSelector implement a transform method that passes the reduced feature subset to the next item in a Pipeline . For example, the method def transform(self, X): return X[:, self.k_feature_idx_] returns the best feature columns, k_feature_idx_ , given a dataset X. Thus, we simply need to construct a Pipeline consisting of the feature selector and the classifier in order to select different feature subsets for different algorithms. During fitting , the optimal feature subsets are automatically determined via the GridSearchCV object, and by calling predict , the fitted feature selector in the pipeline only passes these columns along, which resulted in the best performance for the respective classifier. %%capture --no-display from sklearn import datasets iris = datasets.load_iris() X, y = iris.data[:, :], iris.target from sklearn.model_selection import GridSearchCV from sklearn.linear_model import LogisticRegression from sklearn.naive_bayes import GaussianNB from sklearn.ensemble import RandomForestClassifier from mlxtend.classifier import EnsembleVoteClassifier from sklearn.pipeline import Pipeline from mlxtend.feature_selection import SequentialFeatureSelector clf1 = LogisticRegression(random_state=1) clf2 = RandomForestClassifier(random_state=1) clf3 = GaussianNB() # Creating a feature-selection-classifier pipeline sfs1 = SequentialFeatureSelector(clf1, k_features=4, forward=True, floating=False, scoring='accuracy', verbose=0, cv=0) clf1_pipe = Pipeline([('sfs', sfs1), ('logreg', clf1)]) eclf = EnsembleVoteClassifier(clfs=[clf1_pipe, clf2, clf3], voting='soft') params = {'pipeline__sfs__k_features': [1, 2, 3], 'pipeline__logreg__C': [1.0, 100.0], 'randomforestclassifier__n_estimators': [20, 200]} grid = GridSearchCV(estimator=eclf, param_grid=params, cv=5) grid.fit(iris.data, iris.target) cv_keys = ('mean_test_score', 'std_test_score', 'params') for r, _ in enumerate(grid.cv_results_['mean_test_score']): print(\"%0.3f +/- %0.2f %r\" % (grid.cv_results_[cv_keys[0]][r], grid.cv_results_[cv_keys[1]][r] / 2.0, grid.cv_results_[cv_keys[2]][r])) The best parameters determined via GridSearch are: grid.best_params_ {'pipeline__logreg__C': 1.0, 'pipeline__sfs__k_features': 2, 'randomforestclassifier__n_estimators': 200} Now, we assign these parameters to the ensemble voting classifier, fit the models on the complete training set, and perform a prediction on 3 samples from the Iris dataset. eclf = eclf.set_params(**grid.best_params_) eclf.fit(X, y).predict(X[[1, 51, 149]]) array([0, 1, 2])","title":"Example 3 - Majority voting with classifiers trained on different feature subsets"},{"location":"user_guide/classifier/EnsembleVoteClassifier/#manual-approach","text":"Alternatively, we can select different columns \"manually\" using the ColumnSelector object. In this example, we select only the first (sepal length) and third (petal length) column for the logistic regression classifier ( clf1 ). from mlxtend.feature_selection import ColumnSelector col_sel = ColumnSelector(cols=[0, 2]) clf1_pipe = Pipeline([('sel', col_sel), ('logreg', clf1)]) eclf = EnsembleVoteClassifier(clfs=[clf1_pipe, clf2, clf3], voting='soft') eclf.fit(X, y).predict(X[[1, 51, 149]]) array([0, 1, 2]) Furthermore, we can fit the SequentialFeatureSelector separately, outside the grid search hyperparameter optimization pipeline. Here, we determine the best features first, and then we construct a pipeline using these \"fixed,\" best features as seed for the ColumnSelector : sfs1 = SequentialFeatureSelector(clf1, k_features=2, forward=True, floating=False, scoring='accuracy', verbose=1, cv=0) sfs1.fit(X, y) print('Best features', sfs1.k_feature_idx_) col_sel = ColumnSelector(cols=sfs1.k_feature_idx_) clf1_pipe = Pipeline([('sel', col_sel), ('logreg', clf1)]) [Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers. [Parallel(n_jobs=1)]: Done 4 out of 4 | elapsed: 0.0s finished Features: 1/2[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers. [Parallel(n_jobs=1)]: Done 3 out of 3 | elapsed: 0.0s finished Features: 2/2 Best features (2, 3) eclf = EnsembleVoteClassifier(clfs=[clf1_pipe, clf2, clf3], voting='soft') eclf.fit(X, y).predict(X[[1, 51, 149]]) array([0, 1, 2])","title":"Manual Approach"},{"location":"user_guide/classifier/EnsembleVoteClassifier/#example-5-using-pre-fitted-classifiers","text":"from sklearn import datasets iris = datasets.load_iris() X, y = iris.data[:, 1:3], iris.target Assume that we previously fitted our classifiers: from sklearn import model_selection from sklearn.linear_model import LogisticRegression from sklearn.naive_bayes import GaussianNB from sklearn.ensemble import RandomForestClassifier import numpy as np clf1 = LogisticRegression(random_state=1) clf2 = RandomForestClassifier(random_state=1) clf3 = GaussianNB() for clf in (clf1, clf2, clf3): clf.fit(X, y) By setting fit_base_estimators=False , it will enforce use_clones to be False and the EnsembleVoteClassifier will not re-fit these classifers to save computational time: from mlxtend.classifier import EnsembleVoteClassifier import copy eclf = EnsembleVoteClassifier(clfs=[clf1, clf2, clf3], weights=[1,1,1], fit_base_estimators=False) labels = ['Logistic Regression', 'Random Forest', 'Naive Bayes', 'Ensemble'] eclf.fit(X, y) print('accuracy:', np.mean(y == eclf.predict(X))) accuracy: 0.96 /Users/sebastian/miniforge3/lib/python3.9/site-packages/mlxtend/classifier/ensemble_vote.py:166: UserWarning: fit_base_estimators=False enforces use_clones to be `False` warnings.warn(\"fit_base_estimators=False \" However, please note that fit_base_estimators=False is incompatible to any form of cross-validation that is done in e.g., model_selection.cross_val_score or model_selection.GridSearchCV , etc., since it would require the classifiers to be refit to the training folds. Thus, only use fit_base_estimators=False if you want to make a prediction directly without cross-validation.","title":"Example 5 - Using Pre-fitted Classifiers"},{"location":"user_guide/classifier/EnsembleVoteClassifier/#example-6-ensembles-of-classifiers-that-operate-on-different-feature-subsets","text":"If desired, the different classifiers can be fit to different subsets of features in the training dataset. The following example illustrates how this can be done on a technical level using scikit-learn pipelines and the ColumnSelector : from sklearn.datasets import load_iris from mlxtend.classifier import EnsembleVoteClassifier from mlxtend.feature_selection import ColumnSelector from sklearn.pipeline import make_pipeline from sklearn.linear_model import LogisticRegression iris = load_iris() X = iris.data y = iris.target pipe1 = make_pipeline(ColumnSelector(cols=(0, 2)), LogisticRegression()) pipe2 = make_pipeline(ColumnSelector(cols=(1, 2, 3)), LogisticRegression()) eclf = EnsembleVoteClassifier(clfs=[pipe1, pipe2]) eclf.fit(X, y) EnsembleVoteClassifier(clfs=[Pipeline(steps=[('columnselector', ColumnSelector(cols=(0, 2))), ('logisticregression', LogisticRegression())]), Pipeline(steps=[('columnselector', ColumnSelector(cols=(1, 2, 3))), ('logisticregression', LogisticRegression())])])","title":"Example 6 - Ensembles of Classifiers that Operate on Different Feature Subsets"},{"location":"user_guide/classifier/EnsembleVoteClassifier/#example-7-a-note-about-scikit-learn-svms-and-soft-voting","text":"This section provides some additional technical insights in how probabilities are used when voting='soft' . Note that scikit-learn estimates the probabilities for SVMs (more info here: https://scikit-learn.org/stable/modules/svm.html#scores-probabilities) in a way that these may not be consistent with the class labels that the SVM predicts. This is an extreme example, but let's say we have a dataset with 3 class labels, 0, 1, and 2. For a given training example, the SVM classifier may predict class 2. However, the class-membership probabilities may look as follows: class 0: 99% class 1: 0.5% class 2: 0.5% A practical example of this scenario is shown below: import numpy as np from mlxtend.classifier import EnsembleVoteClassifier from sklearn.svm import SVC from sklearn.datasets import load_iris iris = load_iris() X, y = iris.data, iris.target clf2 = SVC(probability=True, random_state=4) clf2.fit(X, y) eclf = EnsembleVoteClassifier(clfs=[clf2], voting='soft', fit_base_estimators=False) eclf.fit(X, y) for svm_class, e_class, svm_prob, e_prob, in zip(clf2.predict(X), eclf.predict(X), clf2.predict_proba(X), eclf.predict_proba(X)): if svm_class != e_class: print('============') print('Probas from SVM :', svm_prob) print('Class from SVM :', svm_class) print('Probas from SVM in Ensemble:', e_prob) print('Class from SVM in Ensemble :', e_class) print('============') ============ Probas from SVM : [0.00921708 0.49415165 0.49663127] Class from SVM : 1 Probas from SVM in Ensemble: [0.00921708 0.49415165 0.49663127] Class from SVM in Ensemble : 2 ============ /Users/sebastian/miniforge3/lib/python3.9/site-packages/mlxtend/classifier/ensemble_vote.py:166: UserWarning: fit_base_estimators=False enforces use_clones to be `False` warnings.warn(\"fit_base_estimators=False \"","title":"Example 7 - A Note about Scikit-Learn SVMs and Soft Voting"},{"location":"user_guide/classifier/EnsembleVoteClassifier/#example-7-a-note-about-scikit-learn-svms-and-soft-voting_1","text":"Based on the probabilities, we would expect the SVM to predict class 2, because it has the highest probability. Since the EnsembleVoteClassifier uses the argmax function internally if voting='soft' , it would indeed predict class 2 in this case even if the ensemble consists of only one SVM model. Note that in practice, this minor technical detail does not need to concern you, but it is useful to keep it in mind in case you are wondering about results from a 1-model SVM ensemble compared to that SVM alone -- this is not a bug.","title":"Example 7 - A Note about Scikit-Learn SVMs and Soft Voting"},{"location":"user_guide/classifier/EnsembleVoteClassifier/#example-8-optimizing-ensemble-weights-with-nelder-mead","text":"In this section, we will see how we can use a heuristic search method like Nelder-Mead for optimizing the ensemble weights. Suppose we have the following example scenario where we fit 3 individual classifiers on different subsets of the training dataset: from mlxtend.classifier import EnsembleVoteClassifier from mlxtend.data import mnist_data from sklearn.linear_model import LogisticRegression from sklearn.model_selection import train_test_split from sklearn.naive_bayes import GaussianNB from sklearn.tree import DecisionTreeClassifier X, y = mnist_data() X_train, X_val, y_train, y_val = train_test_split( X, y, test_size=0.5, shuffle=True, random_state=1 ) clf1 = GaussianNB() clf2 = LogisticRegression(random_state=123, solver='newton-cg') clf3 = DecisionTreeClassifier(random_state=123, max_depth=2) clf1.fit(X_train[500:1000], y_train[500:1000]) clf2.fit(X_train[750:1250], y_train[750:1250]) clf3.fit(X_train[1250:2000], y_train[1250:2000]); Then, we construct the an ensemble classifier from these 3 classifiers where each classifier contributes equally with weight 1: eclf = EnsembleVoteClassifier( clfs=(clf1, clf2, clf3), voting=\"soft\", # the same would also work with \"hard\" voting weights=(1, 1, 1), use_clones=False, fit_base_estimators=False, ) eclf.fit(X_train, y_train) eclf.score(X_val, y_val) /Users/sebastian/miniforge3/lib/python3.9/site-packages/mlxtend/classifier/ensemble_vote.py:166: UserWarning: fit_base_estimators=False enforces use_clones to be `False` warnings.warn(\"fit_base_estimators=False \" 0.8012 We see that we reach 80% accuracy on the validation set. Can we do better? Maybe they indvidual classifiers shouldn't be contributing equally. Perhaps, we can use an optimization algorothm from scipy.optimize to find a better relative weighting of these individual classifiers. Let's set up an objective function that we want to minimize via SciPy's minimize : from scipy.optimize import minimize def function_to_minimize(weights, fitted_clfs): w1, w2 = weights # these are the new weights! newclf = EnsembleVoteClassifier( voting=\"soft\", use_clones=False, fit_base_estimators=False, clfs=fitted_clfs, weights=(w1, w2, 1.), # use the new weights ) newclf.fit(X_train, y_train) score = newclf.score(X_val, y_val) # change accuracy to error so that smaller is better score_to_minimize = 1 - score return score_to_minimize Note a few things: We only optimize 2 out of the 3 classifier weigths. That's because the weighting is relative to each other, and it would be overkill (and too many degrees of freedom) to also optimize weight 3. We set use_clones=False & fit_base_estimators=False as before, this is to make sure that we use the prefit classifiers in the ensemble classifier. Instead of optimizing the accuracy, we optimize the the classification error, score_to_minimize = 1 - score . That's because we use the minimize function where lower means better. Next, let's choose some initial weight values and run the optimization. Via the bounds we specify the range (lower and upper value) for each weight so that the search doesn't go crazy: %%capture --no-display init_weights = [1., 1.] results = minimize( function_to_minimize, init_weights, args=((clf1, clf2, clf3),), bounds=((0, 5), (0, 5)), method=\"nelder-mead\", ) Let's look at the results! print(results) final_simplex: (array([[0.575 , 1.40625 ], [0.57500153, 1.40622215], [0.57508965, 1.40617647]]), array([0.1324, 0.1324, 0.1324])) fun: 0.13239999999999996 message: 'Optimization terminated successfully.' nfev: 60 nit: 21 status: 0 success: True x: array([0.575 , 1.40625]) It looks like the search was successful and returned the following weights: solution = results[\"x\"] print(solution) [0.575 1.40625] Let's use these new weights in our ensemble classifier: eclf = EnsembleVoteClassifier( clfs=(clf1, clf2, clf3), voting=\"soft\", weights=(solution[0], solution[1], 1), use_clones=False, fit_base_estimators=False, ) eclf.fit(X_train, y_train) eclf.score(X_val, y_val) /Users/sebastian/miniforge3/lib/python3.9/site-packages/mlxtend/classifier/ensemble_vote.py:166: UserWarning: fit_base_estimators=False enforces use_clones to be `False` warnings.warn(\"fit_base_estimators=False \" 0.8676 As we can see, the results on the validation set (0.8676) improved compared to the original ones (0.8012). Yay!","title":"Example 8 - Optimizing Ensemble Weights with Nelder-Mead"},{"location":"user_guide/classifier/EnsembleVoteClassifier/#api","text":"EnsembleVoteClassifier(clfs, voting='hard', weights=None, verbose=0, use_clones=True, fit_base_estimators=True) Soft Voting/Majority Rule classifier for scikit-learn estimators. Parameters clfs : array-like, shape = [n_classifiers] A list of classifiers. Invoking the fit method on the VotingClassifier will fit clones of those original classifiers be stored in the class attribute if use_clones=True (default) and fit_base_estimators=True (default). voting : str, {'hard', 'soft'} (default='hard') If 'hard', uses predicted class labels for majority rule voting. Else if 'soft', predicts the class label based on the argmax of the sums of the predicted probalities, which is recommended for an ensemble of well-calibrated classifiers. weights : array-like, shape = [n_classifiers], optional (default= None ) Sequence of weights ( float or int ) to weight the occurances of predicted class labels ( hard voting) or class probabilities before averaging ( soft voting). Uses uniform weights if None . verbose : int, optional (default=0) Controls the verbosity of the building process. - verbose=0 (default): Prints nothing - verbose=1 : Prints the number & name of the clf being fitted - verbose=2 : Prints info about the parameters of the clf being fitted - verbose>2 : Changes verbose param of the underlying clf to self.verbose - 2 use_clones : bool (default: True) Clones the classifiers for stacking classification if True (default) or else uses the original ones, which will be refitted on the dataset upon calling the fit method. Hence, if use_clones=True, the original input classifiers will remain unmodified upon using the StackingClassifier's fit method. Setting use_clones=False is recommended if you are working with estimators that are supporting the scikit-learn fit/predict API interface but are not compatible to scikit-learn's clone function. fit_base_estimators : bool (default: True) Refits classifiers in clfs if True; uses references to the clfs , otherwise (assumes that the classifiers were already fit). Note: fit_base_estimators=False will enforce use_clones to be False, and is incompatible to most scikit-learn wrappers! For instance, if any form of cross-validation is performed this would require the re-fitting classifiers to training folds, which would raise a NotFitterError if fit_base_estimators=False. (New in mlxtend v0.6.) Attributes classes_ : array-like, shape = [n_predictions] clf : array-like, shape = [n_predictions] The input classifiers; may be overwritten if use_clones=False clf_ : array-like, shape = [n_predictions] Fitted input classifiers; clones if use_clones=True Examples ``` >>> import numpy as np >>> from sklearn.linear_model import LogisticRegression >>> from sklearn.naive_bayes import GaussianNB >>> from sklearn.ensemble import RandomForestClassifier >>> from mlxtend.sklearn import EnsembleVoteClassifier >>> clf1 = LogisticRegression(random_seed=1) >>> clf2 = RandomForestClassifier(random_seed=1) >>> clf3 = GaussianNB() >>> X = np.array([[-1, -1], [-2, -1], [-3, -2], [1, 1], [2, 1], [3, 2]]) >>> y = np.array([1, 1, 1, 2, 2, 2]) >>> eclf1 = EnsembleVoteClassifier(clfs=[clf1, clf2, clf3], ... voting='hard', verbose=1) >>> eclf1 = eclf1.fit(X, y) >>> print(eclf1.predict(X)) [1 1 1 2 2 2] >>> eclf2 = EnsembleVoteClassifier(clfs=[clf1, clf2, clf3], voting='soft') >>> eclf2 = eclf2.fit(X, y) >>> print(eclf2.predict(X)) [1 1 1 2 2 2] >>> eclf3 = EnsembleVoteClassifier(clfs=[clf1, clf2, clf3], ... voting='soft', weights=[2,1,1]) >>> eclf3 = eclf3.fit(X, y) >>> print(eclf3.predict(X)) [1 1 1 2 2 2] >>> For more usage examples, please see https://rasbt.github.io/mlxtend/user_guide/classifier/EnsembleVoteClassifier/ ```","title":"API"},{"location":"user_guide/classifier/EnsembleVoteClassifier/#methods","text":"fit(X, y, sample_weight=None) Learn weight coefficients from training data for each classifier. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values. sample_weight : array-like, shape = [n_samples], optional Sample weights passed as sample_weights to each regressor in the regressors list as well as the meta_regressor. Raises error if some regressor does not support sample_weight in the fit() method. Returns self : object fit_transform(X, y=None, fit_params) Fit to data, then transform it. Fits transformer to `X` and `y` with optional parameters `fit_params` and returns a transformed version of `X`. Parameters X : array-like of shape (n_samples, n_features) Input samples. y : array-like of shape (n_samples,) or (n_samples, n_outputs), default=None Target values (None for unsupervised transformations). **fit_params : dict Additional fit parameters. Returns X_new : ndarray array of shape (n_samples, n_features_new) Transformed array. get_params(deep=True) Return estimator parameter names for GridSearch support. predict(X) Predict class labels for X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns maj : array-like, shape = [n_samples] Predicted class labels. predict_proba(X) Predict class probabilities for X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns avg : array-like, shape = [n_samples, n_classes] Weighted average probability for each class per sample. score(X, y, sample_weight=None) Return the mean accuracy on the given test data and labels. In multi-label classification, this is the subset accuracy which is a harsh metric since you require for each sample that each label set be correctly predicted. Parameters X : array-like of shape (n_samples, n_features) Test samples. y : array-like of shape (n_samples,) or (n_samples, n_outputs) True labels for X . sample_weight : array-like of shape (n_samples,), default=None Sample weights. Returns score : float Mean accuracy of self.predict(X) wrt. y . set_params( params) Set the parameters of this estimator. The method works on simple estimators as well as on nested objects (such as :class:`~sklearn.pipeline.Pipeline`). The latter have parameters of the form ``<component>__<parameter>`` so that it's possible to update each component of a nested object. Parameters **params : dict Estimator parameters. Returns self : estimator instance Estimator instance. transform(X) Return class labels or probabilities for X for each estimator. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns If voting='soft'`` : array-like = [n_classifiers, n_samples, n_classes] Class probabilties calculated by each classifier. If voting='hard'`` : array-like = [n_classifiers, n_samples] Class labels predicted by each classifier. ython","title":"Methods"},{"location":"user_guide/classifier/LogisticRegression/","text":"LogisticRegression: A binary classifier A logistic regression class for binary classification tasks. from mlxtend.classifier import LogisticRegression Overview Related to the Perceptron and 'Adaline' , a Logistic Regression model is a linear model for binary classification. However, instead of minimizing a linear cost function such as the sum of squared errors (SSE) in Adaline, we minimize a sigmoid function, i.e., the logistic function: \\phi(z) = \\frac{1}{1 + e^{-z}}, where z is defined as the net input z = w_0x_0 + w_1x_1 + ... + w_mx_m = \\sum_{j=0}^{m} w_j x_j= \\mathbf{w}^T\\mathbf{x}. The net input is in turn based on the logit function logit(p(y=1 \\mid \\mathbf{x})) = z. Here, p(y=1 \\mid \\mathbf{x}) is the conditional probability that a particular sample belongs to class 1 given its features \\mathbf{x} . The logit function takes inputs in the range [0, 1] and transform them to values over the entire real number range. In contrast, the logistic function takes input values over the entire real number range and transforms them to values in the range [0, 1]. In other words, the logistic function is the inverse of the logit function, and it lets us predict the conditional probability that a certain sample belongs to class 1 (or class 0). After model fitting, the conditional probability p(y=1 \\mid \\mathbf{x}) is converted to a binary class label via a threshold function g(\\cdot) : $$y = g({z}) = \\begin{cases} 1 & \\text{if $\\phi(z) \\ge 0.5$}\\\\ 0 & \\text{otherwise.} \\end{cases} $$ or equivalently: $$y = g({z}) = \\begin{cases} 1 & \\text{if z $\\ge$ 0}\\\\ 0 & \\text{otherwise}. \\end{cases} $$ Objective Function -- Log-Likelihood In order to parameterize a logistic regression model, we maximize the likelihood L(\\cdot) (or minimize the logistic cost function). We write the likelihood as L(\\mathbf{w}) = P(\\mathbf{y} \\mid \\mathbf{x};\\mathbf{w}) = \\prod_{i=1}^{n} P\\big(y^{(i)} \\mid x^{(i)}; \\mathbf{w}\\big) = \\prod^{n}_{i=1}\\bigg(\\phi\\big(z^{(i)}\\big)\\bigg)^{y^{(i)}} \\bigg(1-\\phi\\big(z^{(i)}\\big)\\bigg)^{1-y^{(i)}}, under the assumption that the training samples are independent of each other. In practice, it is easier to maximize the (natural) log of this equation, which is called the log-likelihood function: l(\\mathbf{w}) = \\log L(\\mathbf{w}) = \\sum^{n}_{i=1} y^{(i)} \\log \\bigg(\\phi\\big(z^{(i)}\\big)\\bigg) + \\big( 1 - y^{(i)}\\big) \\log \\big(1-\\phi\\big(z^{(i)}\\big)\\big) One advantage of taking the log is to avoid numeric underflow (and challenges with floating point math) for very small likelihoods. Another advantage is that we can obtain the derivative more easily, using the addition trick to rewrite the product of factors as a summation term, which we can then maximize using optimization algorithms such as gradient ascent. Objective Function -- Logistic Cost Function An alternative to maximizing the log-likelihood, we can define a cost function J(\\cdot) to be minimized; we rewrite the log-likelihood as: J(\\mathbf{w}) = \\sum_{i=1}^{m} - y^{(i)} log \\bigg( \\phi\\big(z^{(i)}\\big) \\bigg) - \\big(1 - y^{(i)}\\big) log\\bigg(1-\\phi\\big(z^{(i)}\\big)\\bigg) $$ J\\big(\\phi(z), y; \\mathbf{w}\\big) = \\begin{cases} -log\\big(\\phi(z) \\big) & \\text{if $y = 1$}\\\\ -log\\big(1- \\phi(z) \\big) & \\text{if $y = 0$} \\end{cases} $$ As we can see in the figure above, we penalize wrong predictions with an increasingly larger cost. Gradient Descent (GD) and Stochastic Gradient Descent (SGD) Optimization Gradient Ascent and the log-likelihood To learn the weight coefficient of a logistic regression model via gradient-based optimization, we compute the partial derivative of the log-likelihood function -- w.r.t. the j th weight -- as follows: \\frac{\\partial}{\\partial w_j} l(\\mathbf{w}) = \\bigg(y \\frac{1}{\\phi(z)} - (1-y) \\frac{1}{1-\\phi{(z)}} \\bigg) \\frac{\\partial}{\\partial w_j}\\phi(z) As an intermediate step, we compute the partial derivative of the sigmoid function, which will come in handy later: \\begin{align} &\\frac{\\partial}{\\partial z} \\phi(z) = \\frac{\\partial}{{\\partial z}} \\frac{1}{1+e^{-z}} \\\\\\\\ &= \\frac{1}{(1 + e^{-z})^{2}} e^{-z}\\\\\\\\ &= \\frac{1}{1+e^{-z}} \\bigg(1 - \\frac{1}{1+e^{-z}} \\bigg)\\\\\\\\ &= \\phi(z)\\big(1-\\phi(z)\\big) \\end{align} Now, we re-substitute \\frac{\\partial}{\\partial z} \\phi(z) = \\phi(z) \\big(1 - \\phi(z)\\big) back into in the log-likelihood partial derivative equation and obtain the equation shown below: \\begin{align} & \\bigg(y \\frac{1}{\\phi{(z)}} - (1 - y) \\frac{1}{1 - \\phi(z)} \\bigg) \\frac{\\partial}{\\partial w_j} \\phi(z) \\\\\\\\ &= \\bigg(y \\frac{1}{\\phi{(z)}} - (1 - y) \\frac{1}{1 - \\phi(z)} \\bigg) \\phi(z) \\big(1 - \\phi(z)\\big) \\frac{\\partial}{\\partial w_j}z\\\\\\\\ &= \\big(y(1-\\phi(z)\\big) - (1 - y) \\phi(z)\\big)x_j\\\\\\\\ &=\\big(y - \\phi(z)\\big)x_j \\end{align} Now, in order to find the weights of the model, we take a step proportional to the positive direction of the gradient to maximize the log-likelihood. Futhermore, we add a coefficient, the learning rate \\eta to the weight update: \\begin{align} & w_j := w_j + \\eta \\frac{\\partial}{\\partial w_j} l(\\mathbf{w})\\\\\\\\ & w_j := w_j + \\eta \\sum^{n}_{i=1} \\big( y^{(i)} - \\phi\\big(z^{(i)}\\big)\\big)x_j^{(i)} \\end{align} Note that the gradient (and weight update) is computed from all samples in the training set in gradient ascent/descent in contrast to stochastic gradient ascent/descent. For more information about the differences between gradient descent and stochastic gradient descent, please see the related article Gradient Descent and Stochastic Gradient Descent . The previous equation shows the weight update for a single weight j . In gradient-based optimization, all weight coefficients are updated simultaneously; the weight update can be written more compactly as \\mathbf{w} := \\mathbf{w} + \\Delta\\mathbf{w}, where \\Delta{\\mathbf{w}} = \\eta \\nabla l(\\mathbf{w}) Gradient Descent and the logistic cost function In the previous section, we derived the gradient of the log-likelihood function, which can be optimized via gradient ascent. Similarly, we can obtain the cost gradient of the logistic cost function J(\\cdot) and minimize it via gradient descent in order to learn the logistic regression model. The update rule for a single weight: \\begin{align} & \\Delta{w_j} = -\\eta \\frac{\\partial J}{\\partial w_j} \\\\ & = - \\eta \\sum_{i=1}^{n}\\big(y^{(i)} - \\phi\\big(z^{(i)}\\big) x^{(i)} \\big) \\end{align} The simultaneous weight update: \\mathbf{w} := \\mathbf{w} + \\Delta\\mathbf{w} where \\Delta{\\mathbf{w}} = - \\eta \\nabla J(\\mathbf{w}). Shuffling Random shuffling is implemented as: for one or more epochs randomly shuffle samples in the training set for training sample i compute gradients and perform weight updates Regularization As a way to tackle overfitting, we can add additional bias to the logistic regression model via a regularization terms. Via the L2 regularization term, we reduce the complexity of the model by penalizing large weight coefficients: L2: \\frac{\\lambda}{2}\\lVert \\mathbf{w} \\lVert_2 = \\frac{\\lambda}{2} \\sum_{j=1}^{m} w_j^2 In order to apply regularization, we just need to add the regularization term to the cost function that we defined for logistic regression to shrink the weights: J(\\mathbf{w}) = \\sum_{i=1}^{m} \\Bigg[ - y^{(i)} log \\bigg( \\phi\\big(z^{(i)}\\big) \\bigg) - \\big(1 - y^{(i)}\\big) log\\bigg(1-\\phi\\big(z^{(i)}\\big)\\bigg) \\Bigg] + \\frac{\\lambda}{2} \\sum_{j=1}^{m} w_j^2 The update rule for a single weight: \\begin{align} & \\Delta{w_j} = -\\eta \\bigg( \\frac{\\partial J}{\\partial w_j} + \\lambda w_j\\bigg)\\\\ & = - \\eta \\sum_{i=1}^{n}\\big(y^{(i)} - \\phi\\big(z^{(i)}\\big) x^{(i)} \\big) - \\eta \\lambda w_j \\end{align} The simultaneous weight update: \\mathbf{w} := \\mathbf{w} + \\Delta\\mathbf{w} where \\Delta{\\mathbf{w}} = - \\eta \\big( \\nabla J(\\mathbf{w}) + \\lambda \\mathbf{w}\\big). For more information on regularization, please see Regularization of Generalized Linear Models . References Bishop, Christopher M. Pattern recognition and machine learning . Springer, 2006. pp. 203-213 Example 1 - Gradient Descent from mlxtend.data import iris_data from mlxtend.plotting import plot_decision_regions from mlxtend.classifier import LogisticRegression import matplotlib.pyplot as plt # Loading Data X, y = iris_data() X = X[:, [0, 3]] # sepal length and petal width X = X[0:100] # class 0 and class 1 y = y[0:100] # class 0 and class 1 # standardize X[:,0] = (X[:,0] - X[:,0].mean()) / X[:,0].std() X[:,1] = (X[:,1] - X[:,1].mean()) / X[:,1].std() lr = LogisticRegression(eta=0.1, l2_lambda=0.0, epochs=100, minibatches=1, # for Gradient Descent random_seed=1, print_progress=3) lr.fit(X, y) plot_decision_regions(X, y, clf=lr) plt.title('Logistic Regression - Gradient Descent') plt.show() plt.plot(range(len(lr.cost_)), lr.cost_) plt.xlabel('Iterations') plt.ylabel('Cost') plt.show() Iteration: 100/100 | Cost 0.32 | Elapsed: 0:00:00 | ETA: 0:00:00 Predicting Class Labels y_pred = lr.predict(X) print('Last 3 Class Labels: %s' % y_pred[-3:]) Last 3 Class Labels: [1 1 1] Predicting Class Probabilities y_pred = lr.predict_proba(X) print('Last 3 Class Labels: %s' % y_pred[-3:]) Last 3 Class Labels: [ 0.99997968 0.99339873 0.99992707] Example 2 - Stochastic Gradient Descent from mlxtend.data import iris_data from mlxtend.plotting import plot_decision_regions from mlxtend.classifier import LogisticRegression import matplotlib.pyplot as plt # Loading Data X, y = iris_data() X = X[:, [0, 3]] # sepal length and petal width X = X[0:100] # class 0 and class 1 y = y[0:100] # class 0 and class 1 # standardize X[:,0] = (X[:,0] - X[:,0].mean()) / X[:,0].std() X[:,1] = (X[:,1] - X[:,1].mean()) / X[:,1].std() lr = LogisticRegression(eta=0.5, epochs=30, l2_lambda=0.0, minibatches=len(y), # for SGD learning random_seed=1, print_progress=3) lr.fit(X, y) plot_decision_regions(X, y, clf=lr) plt.title('Logistic Regression - Stochastic Gradient Descent') plt.show() plt.plot(range(len(lr.cost_)), lr.cost_) plt.xlabel('Iterations') plt.ylabel('Cost') plt.show() Iteration: 30/30 | Cost 0.27 | Elapsed: 0:00:00 | ETA: 0:00:00 Example 3 - Stochastic Gradient Descent w. Minibatches Here, we set minibatches to 5, which will result in Minibatch Learning with a batch size of 20 samples (since 100 Iris samples divided by 5 minibatches equals 20). from mlxtend.data import iris_data from mlxtend.plotting import plot_decision_regions from mlxtend.classifier import LogisticRegression import matplotlib.pyplot as plt # Loading Data X, y = iris_data() X = X[:, [0, 3]] # sepal length and petal width X = X[0:100] # class 0 and class 1 y = y[0:100] # class 0 and class 1 # standardize X[:,0] = (X[:,0] - X[:,0].mean()) / X[:,0].std() X[:,1] = (X[:,1] - X[:,1].mean()) / X[:,1].std() lr = LogisticRegression(eta=0.5, epochs=30, l2_lambda=0.0, minibatches=5, # 100/5 = 20 -> minibatch-s random_seed=1, print_progress=3) lr.fit(X, y) plot_decision_regions(X, y, clf=lr) plt.title('Logistic Regression - Stochastic Gradient Descent') plt.show() plt.plot(range(len(lr.cost_)), lr.cost_) plt.xlabel('Iterations') plt.ylabel('Cost') plt.show() Iteration: 30/30 | Cost 0.25 | Elapsed: 0:00:00 | ETA: 0:00:00 API LogisticRegression(eta=0.01, epochs=50, l2_lambda=0.0, minibatches=1, random_seed=None, print_progress=0) Logistic regression classifier. Note that this implementation of Logistic Regression expects binary class labels in {0, 1}. Parameters eta : float (default: 0.01) Learning rate (between 0.0 and 1.0) epochs : int (default: 50) Passes over the training dataset. Prior to each epoch, the dataset is shuffled if minibatches > 1 to prevent cycles in stochastic gradient descent. l2_lambda : float Regularization parameter for L2 regularization. No regularization if l2_lambda=0.0. minibatches : int (default: 1) The number of minibatches for gradient-based optimization. If 1: Gradient Descent learning If len(y): Stochastic Gradient Descent (SGD) online learning If 1 < minibatches < len(y): SGD Minibatch learning random_seed : int (default: None) Set random state for shuffling and initializing the weights. print_progress : int (default: 0) Prints progress in fitting to stderr. 0: No output 1: Epochs elapsed and cost 2: 1 plus time elapsed 3: 2 plus estimated time until completion Attributes w_ : 2d-array, shape={n_features, 1} Model weights after fitting. b_ : 1d-array, shape={1,} Bias unit after fitting. cost_ : list List of floats with cross_entropy cost (sgd or gd) for every epoch. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/classifier/LogisticRegression/ Methods fit(X, y, init_params=True) Learn model from training data. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values. init_params : bool (default: True) Re-initializes model parameters prior to fitting. Set False to continue training with weights from a previous model fitting. Returns self : object predict(X) Predict targets from X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns target_values : array-like, shape = [n_samples] Predicted target values. predict_proba(X) Predict class probabilities of X from the net input. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns Class 1 probability : float score(X, y) Compute the prediction accuracy Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values (true class labels). Returns acc : float The prediction accuracy as a float between 0.0 and 1.0 (perfect score).","title":"LogisticRegression: A binary classifier"},{"location":"user_guide/classifier/LogisticRegression/#logisticregression-a-binary-classifier","text":"A logistic regression class for binary classification tasks. from mlxtend.classifier import LogisticRegression","title":"LogisticRegression: A binary classifier"},{"location":"user_guide/classifier/LogisticRegression/#overview","text":"Related to the Perceptron and 'Adaline' , a Logistic Regression model is a linear model for binary classification. However, instead of minimizing a linear cost function such as the sum of squared errors (SSE) in Adaline, we minimize a sigmoid function, i.e., the logistic function: \\phi(z) = \\frac{1}{1 + e^{-z}}, where z is defined as the net input z = w_0x_0 + w_1x_1 + ... + w_mx_m = \\sum_{j=0}^{m} w_j x_j= \\mathbf{w}^T\\mathbf{x}. The net input is in turn based on the logit function logit(p(y=1 \\mid \\mathbf{x})) = z. Here, p(y=1 \\mid \\mathbf{x}) is the conditional probability that a particular sample belongs to class 1 given its features \\mathbf{x} . The logit function takes inputs in the range [0, 1] and transform them to values over the entire real number range. In contrast, the logistic function takes input values over the entire real number range and transforms them to values in the range [0, 1]. In other words, the logistic function is the inverse of the logit function, and it lets us predict the conditional probability that a certain sample belongs to class 1 (or class 0). After model fitting, the conditional probability p(y=1 \\mid \\mathbf{x}) is converted to a binary class label via a threshold function g(\\cdot) : $$y = g({z}) = \\begin{cases} 1 & \\text{if $\\phi(z) \\ge 0.5$}\\\\ 0 & \\text{otherwise.} \\end{cases} $$ or equivalently: $$y = g({z}) = \\begin{cases} 1 & \\text{if z $\\ge$ 0}\\\\ 0 & \\text{otherwise}. \\end{cases} $$","title":"Overview"},{"location":"user_guide/classifier/LogisticRegression/#objective-function-log-likelihood","text":"In order to parameterize a logistic regression model, we maximize the likelihood L(\\cdot) (or minimize the logistic cost function). We write the likelihood as L(\\mathbf{w}) = P(\\mathbf{y} \\mid \\mathbf{x};\\mathbf{w}) = \\prod_{i=1}^{n} P\\big(y^{(i)} \\mid x^{(i)}; \\mathbf{w}\\big) = \\prod^{n}_{i=1}\\bigg(\\phi\\big(z^{(i)}\\big)\\bigg)^{y^{(i)}} \\bigg(1-\\phi\\big(z^{(i)}\\big)\\bigg)^{1-y^{(i)}}, under the assumption that the training samples are independent of each other. In practice, it is easier to maximize the (natural) log of this equation, which is called the log-likelihood function: l(\\mathbf{w}) = \\log L(\\mathbf{w}) = \\sum^{n}_{i=1} y^{(i)} \\log \\bigg(\\phi\\big(z^{(i)}\\big)\\bigg) + \\big( 1 - y^{(i)}\\big) \\log \\big(1-\\phi\\big(z^{(i)}\\big)\\big) One advantage of taking the log is to avoid numeric underflow (and challenges with floating point math) for very small likelihoods. Another advantage is that we can obtain the derivative more easily, using the addition trick to rewrite the product of factors as a summation term, which we can then maximize using optimization algorithms such as gradient ascent.","title":"Objective Function -- Log-Likelihood"},{"location":"user_guide/classifier/LogisticRegression/#objective-function-logistic-cost-function","text":"An alternative to maximizing the log-likelihood, we can define a cost function J(\\cdot) to be minimized; we rewrite the log-likelihood as: J(\\mathbf{w}) = \\sum_{i=1}^{m} - y^{(i)} log \\bigg( \\phi\\big(z^{(i)}\\big) \\bigg) - \\big(1 - y^{(i)}\\big) log\\bigg(1-\\phi\\big(z^{(i)}\\big)\\bigg) $$ J\\big(\\phi(z), y; \\mathbf{w}\\big) = \\begin{cases} -log\\big(\\phi(z) \\big) & \\text{if $y = 1$}\\\\ -log\\big(1- \\phi(z) \\big) & \\text{if $y = 0$} \\end{cases} $$ As we can see in the figure above, we penalize wrong predictions with an increasingly larger cost.","title":"Objective Function -- Logistic Cost Function"},{"location":"user_guide/classifier/LogisticRegression/#gradient-descent-gd-and-stochastic-gradient-descent-sgd-optimization","text":"","title":"Gradient Descent (GD)  and Stochastic Gradient Descent (SGD) Optimization"},{"location":"user_guide/classifier/LogisticRegression/#gradient-ascent-and-the-log-likelihood","text":"To learn the weight coefficient of a logistic regression model via gradient-based optimization, we compute the partial derivative of the log-likelihood function -- w.r.t. the j th weight -- as follows: \\frac{\\partial}{\\partial w_j} l(\\mathbf{w}) = \\bigg(y \\frac{1}{\\phi(z)} - (1-y) \\frac{1}{1-\\phi{(z)}} \\bigg) \\frac{\\partial}{\\partial w_j}\\phi(z) As an intermediate step, we compute the partial derivative of the sigmoid function, which will come in handy later: \\begin{align} &\\frac{\\partial}{\\partial z} \\phi(z) = \\frac{\\partial}{{\\partial z}} \\frac{1}{1+e^{-z}} \\\\\\\\ &= \\frac{1}{(1 + e^{-z})^{2}} e^{-z}\\\\\\\\ &= \\frac{1}{1+e^{-z}} \\bigg(1 - \\frac{1}{1+e^{-z}} \\bigg)\\\\\\\\ &= \\phi(z)\\big(1-\\phi(z)\\big) \\end{align} Now, we re-substitute \\frac{\\partial}{\\partial z} \\phi(z) = \\phi(z) \\big(1 - \\phi(z)\\big) back into in the log-likelihood partial derivative equation and obtain the equation shown below: \\begin{align} & \\bigg(y \\frac{1}{\\phi{(z)}} - (1 - y) \\frac{1}{1 - \\phi(z)} \\bigg) \\frac{\\partial}{\\partial w_j} \\phi(z) \\\\\\\\ &= \\bigg(y \\frac{1}{\\phi{(z)}} - (1 - y) \\frac{1}{1 - \\phi(z)} \\bigg) \\phi(z) \\big(1 - \\phi(z)\\big) \\frac{\\partial}{\\partial w_j}z\\\\\\\\ &= \\big(y(1-\\phi(z)\\big) - (1 - y) \\phi(z)\\big)x_j\\\\\\\\ &=\\big(y - \\phi(z)\\big)x_j \\end{align} Now, in order to find the weights of the model, we take a step proportional to the positive direction of the gradient to maximize the log-likelihood. Futhermore, we add a coefficient, the learning rate \\eta to the weight update: \\begin{align} & w_j := w_j + \\eta \\frac{\\partial}{\\partial w_j} l(\\mathbf{w})\\\\\\\\ & w_j := w_j + \\eta \\sum^{n}_{i=1} \\big( y^{(i)} - \\phi\\big(z^{(i)}\\big)\\big)x_j^{(i)} \\end{align} Note that the gradient (and weight update) is computed from all samples in the training set in gradient ascent/descent in contrast to stochastic gradient ascent/descent. For more information about the differences between gradient descent and stochastic gradient descent, please see the related article Gradient Descent and Stochastic Gradient Descent . The previous equation shows the weight update for a single weight j . In gradient-based optimization, all weight coefficients are updated simultaneously; the weight update can be written more compactly as \\mathbf{w} := \\mathbf{w} + \\Delta\\mathbf{w}, where \\Delta{\\mathbf{w}} = \\eta \\nabla l(\\mathbf{w})","title":"Gradient Ascent and the log-likelihood"},{"location":"user_guide/classifier/LogisticRegression/#gradient-descent-and-the-logistic-cost-function","text":"In the previous section, we derived the gradient of the log-likelihood function, which can be optimized via gradient ascent. Similarly, we can obtain the cost gradient of the logistic cost function J(\\cdot) and minimize it via gradient descent in order to learn the logistic regression model. The update rule for a single weight: \\begin{align} & \\Delta{w_j} = -\\eta \\frac{\\partial J}{\\partial w_j} \\\\ & = - \\eta \\sum_{i=1}^{n}\\big(y^{(i)} - \\phi\\big(z^{(i)}\\big) x^{(i)} \\big) \\end{align} The simultaneous weight update: \\mathbf{w} := \\mathbf{w} + \\Delta\\mathbf{w} where \\Delta{\\mathbf{w}} = - \\eta \\nabla J(\\mathbf{w}).","title":"Gradient Descent and the logistic cost function"},{"location":"user_guide/classifier/LogisticRegression/#shuffling","text":"Random shuffling is implemented as: for one or more epochs randomly shuffle samples in the training set for training sample i compute gradients and perform weight updates","title":"Shuffling"},{"location":"user_guide/classifier/LogisticRegression/#regularization","text":"As a way to tackle overfitting, we can add additional bias to the logistic regression model via a regularization terms. Via the L2 regularization term, we reduce the complexity of the model by penalizing large weight coefficients: L2: \\frac{\\lambda}{2}\\lVert \\mathbf{w} \\lVert_2 = \\frac{\\lambda}{2} \\sum_{j=1}^{m} w_j^2 In order to apply regularization, we just need to add the regularization term to the cost function that we defined for logistic regression to shrink the weights: J(\\mathbf{w}) = \\sum_{i=1}^{m} \\Bigg[ - y^{(i)} log \\bigg( \\phi\\big(z^{(i)}\\big) \\bigg) - \\big(1 - y^{(i)}\\big) log\\bigg(1-\\phi\\big(z^{(i)}\\big)\\bigg) \\Bigg] + \\frac{\\lambda}{2} \\sum_{j=1}^{m} w_j^2 The update rule for a single weight: \\begin{align} & \\Delta{w_j} = -\\eta \\bigg( \\frac{\\partial J}{\\partial w_j} + \\lambda w_j\\bigg)\\\\ & = - \\eta \\sum_{i=1}^{n}\\big(y^{(i)} - \\phi\\big(z^{(i)}\\big) x^{(i)} \\big) - \\eta \\lambda w_j \\end{align} The simultaneous weight update: \\mathbf{w} := \\mathbf{w} + \\Delta\\mathbf{w} where \\Delta{\\mathbf{w}} = - \\eta \\big( \\nabla J(\\mathbf{w}) + \\lambda \\mathbf{w}\\big). For more information on regularization, please see Regularization of Generalized Linear Models .","title":"Regularization"},{"location":"user_guide/classifier/LogisticRegression/#references","text":"Bishop, Christopher M. Pattern recognition and machine learning . Springer, 2006. pp. 203-213","title":"References"},{"location":"user_guide/classifier/LogisticRegression/#example-1-gradient-descent","text":"from mlxtend.data import iris_data from mlxtend.plotting import plot_decision_regions from mlxtend.classifier import LogisticRegression import matplotlib.pyplot as plt # Loading Data X, y = iris_data() X = X[:, [0, 3]] # sepal length and petal width X = X[0:100] # class 0 and class 1 y = y[0:100] # class 0 and class 1 # standardize X[:,0] = (X[:,0] - X[:,0].mean()) / X[:,0].std() X[:,1] = (X[:,1] - X[:,1].mean()) / X[:,1].std() lr = LogisticRegression(eta=0.1, l2_lambda=0.0, epochs=100, minibatches=1, # for Gradient Descent random_seed=1, print_progress=3) lr.fit(X, y) plot_decision_regions(X, y, clf=lr) plt.title('Logistic Regression - Gradient Descent') plt.show() plt.plot(range(len(lr.cost_)), lr.cost_) plt.xlabel('Iterations') plt.ylabel('Cost') plt.show() Iteration: 100/100 | Cost 0.32 | Elapsed: 0:00:00 | ETA: 0:00:00","title":"Example 1 - Gradient Descent"},{"location":"user_guide/classifier/LogisticRegression/#predicting-class-labels","text":"y_pred = lr.predict(X) print('Last 3 Class Labels: %s' % y_pred[-3:]) Last 3 Class Labels: [1 1 1]","title":"Predicting Class Labels"},{"location":"user_guide/classifier/LogisticRegression/#predicting-class-probabilities","text":"y_pred = lr.predict_proba(X) print('Last 3 Class Labels: %s' % y_pred[-3:]) Last 3 Class Labels: [ 0.99997968 0.99339873 0.99992707]","title":"Predicting Class Probabilities"},{"location":"user_guide/classifier/LogisticRegression/#example-2-stochastic-gradient-descent","text":"from mlxtend.data import iris_data from mlxtend.plotting import plot_decision_regions from mlxtend.classifier import LogisticRegression import matplotlib.pyplot as plt # Loading Data X, y = iris_data() X = X[:, [0, 3]] # sepal length and petal width X = X[0:100] # class 0 and class 1 y = y[0:100] # class 0 and class 1 # standardize X[:,0] = (X[:,0] - X[:,0].mean()) / X[:,0].std() X[:,1] = (X[:,1] - X[:,1].mean()) / X[:,1].std() lr = LogisticRegression(eta=0.5, epochs=30, l2_lambda=0.0, minibatches=len(y), # for SGD learning random_seed=1, print_progress=3) lr.fit(X, y) plot_decision_regions(X, y, clf=lr) plt.title('Logistic Regression - Stochastic Gradient Descent') plt.show() plt.plot(range(len(lr.cost_)), lr.cost_) plt.xlabel('Iterations') plt.ylabel('Cost') plt.show() Iteration: 30/30 | Cost 0.27 | Elapsed: 0:00:00 | ETA: 0:00:00","title":"Example 2 - Stochastic Gradient Descent"},{"location":"user_guide/classifier/LogisticRegression/#example-3-stochastic-gradient-descent-w-minibatches","text":"Here, we set minibatches to 5, which will result in Minibatch Learning with a batch size of 20 samples (since 100 Iris samples divided by 5 minibatches equals 20). from mlxtend.data import iris_data from mlxtend.plotting import plot_decision_regions from mlxtend.classifier import LogisticRegression import matplotlib.pyplot as plt # Loading Data X, y = iris_data() X = X[:, [0, 3]] # sepal length and petal width X = X[0:100] # class 0 and class 1 y = y[0:100] # class 0 and class 1 # standardize X[:,0] = (X[:,0] - X[:,0].mean()) / X[:,0].std() X[:,1] = (X[:,1] - X[:,1].mean()) / X[:,1].std() lr = LogisticRegression(eta=0.5, epochs=30, l2_lambda=0.0, minibatches=5, # 100/5 = 20 -> minibatch-s random_seed=1, print_progress=3) lr.fit(X, y) plot_decision_regions(X, y, clf=lr) plt.title('Logistic Regression - Stochastic Gradient Descent') plt.show() plt.plot(range(len(lr.cost_)), lr.cost_) plt.xlabel('Iterations') plt.ylabel('Cost') plt.show() Iteration: 30/30 | Cost 0.25 | Elapsed: 0:00:00 | ETA: 0:00:00","title":"Example 3 - Stochastic Gradient Descent w. Minibatches"},{"location":"user_guide/classifier/LogisticRegression/#api","text":"LogisticRegression(eta=0.01, epochs=50, l2_lambda=0.0, minibatches=1, random_seed=None, print_progress=0) Logistic regression classifier. Note that this implementation of Logistic Regression expects binary class labels in {0, 1}. Parameters eta : float (default: 0.01) Learning rate (between 0.0 and 1.0) epochs : int (default: 50) Passes over the training dataset. Prior to each epoch, the dataset is shuffled if minibatches > 1 to prevent cycles in stochastic gradient descent. l2_lambda : float Regularization parameter for L2 regularization. No regularization if l2_lambda=0.0. minibatches : int (default: 1) The number of minibatches for gradient-based optimization. If 1: Gradient Descent learning If len(y): Stochastic Gradient Descent (SGD) online learning If 1 < minibatches < len(y): SGD Minibatch learning random_seed : int (default: None) Set random state for shuffling and initializing the weights. print_progress : int (default: 0) Prints progress in fitting to stderr. 0: No output 1: Epochs elapsed and cost 2: 1 plus time elapsed 3: 2 plus estimated time until completion Attributes w_ : 2d-array, shape={n_features, 1} Model weights after fitting. b_ : 1d-array, shape={1,} Bias unit after fitting. cost_ : list List of floats with cross_entropy cost (sgd or gd) for every epoch. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/classifier/LogisticRegression/","title":"API"},{"location":"user_guide/classifier/LogisticRegression/#methods","text":"fit(X, y, init_params=True) Learn model from training data. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values. init_params : bool (default: True) Re-initializes model parameters prior to fitting. Set False to continue training with weights from a previous model fitting. Returns self : object predict(X) Predict targets from X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns target_values : array-like, shape = [n_samples] Predicted target values. predict_proba(X) Predict class probabilities of X from the net input. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns Class 1 probability : float score(X, y) Compute the prediction accuracy Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values (true class labels). Returns acc : float The prediction accuracy as a float between 0.0 and 1.0 (perfect score).","title":"Methods"},{"location":"user_guide/classifier/MultiLayerPerceptron/","text":"MultilayerPerceptron: A simple multilayer neural network Implementation of a multilayer perceptron, a feedforward artificial neural network. from mlxtend.classifier import MultiLayerPerceptron Overview Although the code is fully working and can be used for common classification tasks, this implementation is not geared towards efficiency but clarity \u2013 the original code was written for demonstration purposes. Basic Architecture The neurons x_0 and a_0 represent the bias units ( x_0=1 , a_0=1 ). The i th superscript denotes the i th layer, and the j th subscripts stands for the index of the respective unit. For example, a_{1}^{(2)} refers to the first activation unit after the bias unit (i.e., 2nd activation unit) in the 2nd layer (here: the hidden layer) \\begin{align} \\mathbf{a^{(2)}} &= \\begin{bmatrix} a_{0}^{(2)} \\\\ a_{1}^{(2)} \\\\ \\vdots \\\\ a_{m}^{(2)} \\end{bmatrix}. \\end{align} Each layer (l) in a multi-layer perceptron, a directed graph, is fully connected to the next layer (l+1) . We write the weight coefficient that connects the k th unit in the l th layer to the j th unit in layer l+1 as w^{(l)}_{j, k} . For example, the weight coefficient that connects the units a_0^{(2)} \\rightarrow a_1^{(3)} would be written as w_{1,0}^{(2)} . Activation In the current implementation, the activations of the hidden layer(s) are computed via the logistic (sigmoid) function \\phi(z) = \\frac{1}{1 + e^{-z}}. (For more details on the logistic function, please see classifier.LogisticRegression ; a general overview of different activation function can be found here .) Furthermore, the MLP uses the softmax function in the output layer, For more details on the logistic function, please see classifier.SoftmaxRegression . References D. R. G. H. R. Williams and G. Hinton. Learning representations by back-propagating errors . Nature, pages 323\u2013533, 1986. C. M. Bishop. Neural networks for pattern recognition . Oxford University Press, 1995. T. Hastie, J. Friedman, and R. Tibshirani. The Elements of Statistical Learning , Volume 2. Springer, 2009. Example 1 - Classifying Iris Flowers Load 2 features from Iris (petal length and petal width) for visualization purposes: from mlxtend.data import iris_data X, y = iris_data() X = X[:, [0, 3]] # standardize training data X_std = (X - X.mean(axis=0)) / X.std(axis=0) Train neural network for 3 output flower classes ('Setosa', 'Versicolor', 'Virginica'), regular gradient decent ( minibatches=1 ), 30 hidden units, and no regularization. Gradient Descent Setting the minibatches to 1 will result in gradient descent training; please see Gradient Descent vs. Stochastic Gradient Descent for details. from mlxtend.classifier import MultiLayerPerceptron as MLP nn1 = MLP(hidden_layers=[50], l2=0.00, l1=0.0, epochs=150, eta=0.05, momentum=0.1, decrease_const=0.0, minibatches=1, random_seed=1, print_progress=3) nn1 = nn1.fit(X_std, y) Iteration: 150/150 | Cost 0.06 | Elapsed: 0:00:00 | ETA: 0:00:00 from mlxtend.plotting import plot_decision_regions import matplotlib.pyplot as plt fig = plot_decision_regions(X=X_std, y=y, clf=nn1, legend=2) plt.title('Multi-layer Perceptron w. 1 hidden layer (logistic sigmoid)') plt.show() import matplotlib.pyplot as plt plt.plot(range(len(nn1.cost_)), nn1.cost_) plt.ylabel('Cost') plt.xlabel('Epochs') plt.show() print('Accuracy: %.2f%%' % (100 * nn1.score(X_std, y))) Accuracy: 96.67% Stochastic Gradient Descent Setting minibatches to n_samples will result in stochastic gradient descent training; please see Gradient Descent vs. Stochastic Gradient Descent for details. nn2 = MLP(hidden_layers=[50], l2=0.00, l1=0.0, epochs=5, eta=0.005, momentum=0.1, decrease_const=0.0, minibatches=len(y), random_seed=1, print_progress=3) nn2.fit(X_std, y) plt.plot(range(len(nn2.cost_)), nn2.cost_) plt.ylabel('Cost') plt.xlabel('Epochs') plt.show() Iteration: 5/5 | Cost 0.11 | Elapsed: 00:00:00 | ETA: 00:00:00 Continue the training for 25 epochs... nn2.epochs = 25 nn2 = nn2.fit(X_std, y) Iteration: 25/25 | Cost 0.07 | Elapsed: 0:00:00 | ETA: 0:00:00 plt.plot(range(len(nn2.cost_)), nn2.cost_) plt.ylabel('Cost') plt.xlabel('Epochs') plt.show() Example 2 - Classifying Handwritten Digits from a 10% MNIST Subset Load a 5000-sample subset of the MNIST dataset (please see data.loadlocal_mnist if you want to download and read in the complete MNIST dataset). from mlxtend.data import mnist_data from mlxtend.preprocessing import shuffle_arrays_unison X, y = mnist_data() X, y = shuffle_arrays_unison((X, y), random_seed=1) X_train, y_train = X[:500], y[:500] X_test, y_test = X[500:], y[500:] Visualize a sample from the MNIST dataset to check if it was loaded correctly: import matplotlib.pyplot as plt def plot_digit(X, y, idx): img = X[idx].reshape(28,28) plt.imshow(img, cmap='Greys', interpolation='nearest') plt.title('true label: %d' % y[idx]) plt.show() plot_digit(X, y, 3500) Standardize pixel values: import numpy as np from mlxtend.preprocessing import standardize X_train_std, params = standardize(X_train, columns=range(X_train.shape[1]), return_params=True) X_test_std = standardize(X_test, columns=range(X_test.shape[1]), params=params) Initialize the neural network to recognize the 10 different digits (0-10) using 300 epochs and mini-batch learning. nn1 = MLP(hidden_layers=[150], l2=0.00, l1=0.0, epochs=100, eta=0.005, momentum=0.0, decrease_const=0.0, minibatches=100, random_seed=1, print_progress=3) Learn the features while printing the progress to get an idea about how long it may take. import matplotlib.pyplot as plt nn1.fit(X_train_std, y_train) plt.plot(range(len(nn1.cost_)), nn1.cost_) plt.ylabel('Cost') plt.xlabel('Epochs') plt.show() Iteration: 100/100 | Cost 0.01 | Elapsed: 0:00:17 | ETA: 0:00:00 print('Train Accuracy: %.2f%%' % (100 * nn1.score(X_train_std, y_train))) print('Test Accuracy: %.2f%%' % (100 * nn1.score(X_test_std, y_test))) Train Accuracy: 100.00% Test Accuracy: 84.62% Please note that this neural network has been trained on only 10% of the MNIST data for technical demonstration purposes, hence, the lousy predictive performance. API MultiLayerPerceptron(eta=0.5, epochs=50, hidden_layers=[50], n_classes=None, momentum=0.0, l1=0.0, l2=0.0, dropout=1.0, decrease_const=0.0, minibatches=1, random_seed=None, print_progress=0) Multi-layer perceptron classifier with logistic sigmoid activations Parameters eta : float (default: 0.5) Learning rate (between 0.0 and 1.0) epochs : int (default: 50) Passes over the training dataset. Prior to each epoch, the dataset is shuffled if minibatches > 1 to prevent cycles in stochastic gradient descent. hidden_layers : list (default: [50]) Number of units per hidden layer. By default 50 units in the first hidden layer. At the moment only 1 hidden layer is supported n_classes : int (default: None) A positive integer to declare the number of class labels if not all class labels are present in a partial training set. Gets the number of class labels automatically if None. l1 : float (default: 0.0) L1 regularization strength l2 : float (default: 0.0) L2 regularization strength momentum : float (default: 0.0) Momentum constant. Factor multiplied with the gradient of the previous epoch t-1 to improve learning speed w(t) := w(t) - (grad(t) + momentum * grad(t-1)) decrease_const : float (default: 0.0) Decrease constant. Shrinks the learning rate after each epoch via eta / (1 + epoch*decrease_const) minibatches : int (default: 1) Divide the training data into k minibatches for accelerated stochastic gradient descent learning. Gradient Descent Learning if minibatches = 1 Stochastic Gradient Descent learning if minibatches = len(y) Minibatch learning if minibatches > 1 random_seed : int (default: None) Set random state for shuffling and initializing the weights. print_progress : int (default: 0) Prints progress in fitting to stderr. 0: No output 1: Epochs elapsed and cost 2: 1 plus time elapsed 3: 2 plus estimated time until completion Attributes w_ : 2d-array, shape=[n_features, n_classes] Weights after fitting. b_ : 1D-array, shape=[n_classes] Bias units after fitting. cost_ : list List of floats; the mean categorical cross entropy cost after each epoch. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/classifier/MultiLayerPerceptron/ Methods fit(X, y, init_params=True) Learn model from training data. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values. init_params : bool (default: True) Re-initializes model parameters prior to fitting. Set False to continue training with weights from a previous model fitting. Returns self : object predict(X) Predict targets from X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns target_values : array-like, shape = [n_samples] Predicted target values. predict_proba(X) Predict class probabilities of X from the net input. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns Class probabilties : array-like, shape= [n_samples, n_classes] score(X, y) Compute the prediction accuracy Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values (true class labels). Returns acc : float The prediction accuracy as a float between 0.0 and 1.0 (perfect score). ython","title":"MultilayerPerceptron: A simple multilayer neural network"},{"location":"user_guide/classifier/MultiLayerPerceptron/#multilayerperceptron-a-simple-multilayer-neural-network","text":"Implementation of a multilayer perceptron, a feedforward artificial neural network. from mlxtend.classifier import MultiLayerPerceptron","title":"MultilayerPerceptron: A simple multilayer neural network"},{"location":"user_guide/classifier/MultiLayerPerceptron/#overview","text":"Although the code is fully working and can be used for common classification tasks, this implementation is not geared towards efficiency but clarity \u2013 the original code was written for demonstration purposes.","title":"Overview"},{"location":"user_guide/classifier/MultiLayerPerceptron/#basic-architecture","text":"The neurons x_0 and a_0 represent the bias units ( x_0=1 , a_0=1 ). The i th superscript denotes the i th layer, and the j th subscripts stands for the index of the respective unit. For example, a_{1}^{(2)} refers to the first activation unit after the bias unit (i.e., 2nd activation unit) in the 2nd layer (here: the hidden layer) \\begin{align} \\mathbf{a^{(2)}} &= \\begin{bmatrix} a_{0}^{(2)} \\\\ a_{1}^{(2)} \\\\ \\vdots \\\\ a_{m}^{(2)} \\end{bmatrix}. \\end{align} Each layer (l) in a multi-layer perceptron, a directed graph, is fully connected to the next layer (l+1) . We write the weight coefficient that connects the k th unit in the l th layer to the j th unit in layer l+1 as w^{(l)}_{j, k} . For example, the weight coefficient that connects the units a_0^{(2)} \\rightarrow a_1^{(3)} would be written as w_{1,0}^{(2)} .","title":"Basic Architecture"},{"location":"user_guide/classifier/MultiLayerPerceptron/#activation","text":"In the current implementation, the activations of the hidden layer(s) are computed via the logistic (sigmoid) function \\phi(z) = \\frac{1}{1 + e^{-z}}. (For more details on the logistic function, please see classifier.LogisticRegression ; a general overview of different activation function can be found here .) Furthermore, the MLP uses the softmax function in the output layer, For more details on the logistic function, please see classifier.SoftmaxRegression .","title":"Activation"},{"location":"user_guide/classifier/MultiLayerPerceptron/#references","text":"D. R. G. H. R. Williams and G. Hinton. Learning representations by back-propagating errors . Nature, pages 323\u2013533, 1986. C. M. Bishop. Neural networks for pattern recognition . Oxford University Press, 1995. T. Hastie, J. Friedman, and R. Tibshirani. The Elements of Statistical Learning , Volume 2. Springer, 2009.","title":"References"},{"location":"user_guide/classifier/MultiLayerPerceptron/#example-1-classifying-iris-flowers","text":"Load 2 features from Iris (petal length and petal width) for visualization purposes: from mlxtend.data import iris_data X, y = iris_data() X = X[:, [0, 3]] # standardize training data X_std = (X - X.mean(axis=0)) / X.std(axis=0) Train neural network for 3 output flower classes ('Setosa', 'Versicolor', 'Virginica'), regular gradient decent ( minibatches=1 ), 30 hidden units, and no regularization.","title":"Example 1 - Classifying Iris Flowers"},{"location":"user_guide/classifier/MultiLayerPerceptron/#gradient-descent","text":"Setting the minibatches to 1 will result in gradient descent training; please see Gradient Descent vs. Stochastic Gradient Descent for details. from mlxtend.classifier import MultiLayerPerceptron as MLP nn1 = MLP(hidden_layers=[50], l2=0.00, l1=0.0, epochs=150, eta=0.05, momentum=0.1, decrease_const=0.0, minibatches=1, random_seed=1, print_progress=3) nn1 = nn1.fit(X_std, y) Iteration: 150/150 | Cost 0.06 | Elapsed: 0:00:00 | ETA: 0:00:00 from mlxtend.plotting import plot_decision_regions import matplotlib.pyplot as plt fig = plot_decision_regions(X=X_std, y=y, clf=nn1, legend=2) plt.title('Multi-layer Perceptron w. 1 hidden layer (logistic sigmoid)') plt.show() import matplotlib.pyplot as plt plt.plot(range(len(nn1.cost_)), nn1.cost_) plt.ylabel('Cost') plt.xlabel('Epochs') plt.show() print('Accuracy: %.2f%%' % (100 * nn1.score(X_std, y))) Accuracy: 96.67%","title":"Gradient Descent"},{"location":"user_guide/classifier/MultiLayerPerceptron/#stochastic-gradient-descent","text":"Setting minibatches to n_samples will result in stochastic gradient descent training; please see Gradient Descent vs. Stochastic Gradient Descent for details. nn2 = MLP(hidden_layers=[50], l2=0.00, l1=0.0, epochs=5, eta=0.005, momentum=0.1, decrease_const=0.0, minibatches=len(y), random_seed=1, print_progress=3) nn2.fit(X_std, y) plt.plot(range(len(nn2.cost_)), nn2.cost_) plt.ylabel('Cost') plt.xlabel('Epochs') plt.show() Iteration: 5/5 | Cost 0.11 | Elapsed: 00:00:00 | ETA: 00:00:00 Continue the training for 25 epochs... nn2.epochs = 25 nn2 = nn2.fit(X_std, y) Iteration: 25/25 | Cost 0.07 | Elapsed: 0:00:00 | ETA: 0:00:00 plt.plot(range(len(nn2.cost_)), nn2.cost_) plt.ylabel('Cost') plt.xlabel('Epochs') plt.show()","title":"Stochastic Gradient Descent"},{"location":"user_guide/classifier/MultiLayerPerceptron/#example-2-classifying-handwritten-digits-from-a-10-mnist-subset","text":"Load a 5000-sample subset of the MNIST dataset (please see data.loadlocal_mnist if you want to download and read in the complete MNIST dataset). from mlxtend.data import mnist_data from mlxtend.preprocessing import shuffle_arrays_unison X, y = mnist_data() X, y = shuffle_arrays_unison((X, y), random_seed=1) X_train, y_train = X[:500], y[:500] X_test, y_test = X[500:], y[500:] Visualize a sample from the MNIST dataset to check if it was loaded correctly: import matplotlib.pyplot as plt def plot_digit(X, y, idx): img = X[idx].reshape(28,28) plt.imshow(img, cmap='Greys', interpolation='nearest') plt.title('true label: %d' % y[idx]) plt.show() plot_digit(X, y, 3500) Standardize pixel values: import numpy as np from mlxtend.preprocessing import standardize X_train_std, params = standardize(X_train, columns=range(X_train.shape[1]), return_params=True) X_test_std = standardize(X_test, columns=range(X_test.shape[1]), params=params) Initialize the neural network to recognize the 10 different digits (0-10) using 300 epochs and mini-batch learning. nn1 = MLP(hidden_layers=[150], l2=0.00, l1=0.0, epochs=100, eta=0.005, momentum=0.0, decrease_const=0.0, minibatches=100, random_seed=1, print_progress=3) Learn the features while printing the progress to get an idea about how long it may take. import matplotlib.pyplot as plt nn1.fit(X_train_std, y_train) plt.plot(range(len(nn1.cost_)), nn1.cost_) plt.ylabel('Cost') plt.xlabel('Epochs') plt.show() Iteration: 100/100 | Cost 0.01 | Elapsed: 0:00:17 | ETA: 0:00:00 print('Train Accuracy: %.2f%%' % (100 * nn1.score(X_train_std, y_train))) print('Test Accuracy: %.2f%%' % (100 * nn1.score(X_test_std, y_test))) Train Accuracy: 100.00% Test Accuracy: 84.62% Please note that this neural network has been trained on only 10% of the MNIST data for technical demonstration purposes, hence, the lousy predictive performance.","title":"Example 2 - Classifying Handwritten Digits from a 10% MNIST Subset"},{"location":"user_guide/classifier/MultiLayerPerceptron/#api","text":"MultiLayerPerceptron(eta=0.5, epochs=50, hidden_layers=[50], n_classes=None, momentum=0.0, l1=0.0, l2=0.0, dropout=1.0, decrease_const=0.0, minibatches=1, random_seed=None, print_progress=0) Multi-layer perceptron classifier with logistic sigmoid activations Parameters eta : float (default: 0.5) Learning rate (between 0.0 and 1.0) epochs : int (default: 50) Passes over the training dataset. Prior to each epoch, the dataset is shuffled if minibatches > 1 to prevent cycles in stochastic gradient descent. hidden_layers : list (default: [50]) Number of units per hidden layer. By default 50 units in the first hidden layer. At the moment only 1 hidden layer is supported n_classes : int (default: None) A positive integer to declare the number of class labels if not all class labels are present in a partial training set. Gets the number of class labels automatically if None. l1 : float (default: 0.0) L1 regularization strength l2 : float (default: 0.0) L2 regularization strength momentum : float (default: 0.0) Momentum constant. Factor multiplied with the gradient of the previous epoch t-1 to improve learning speed w(t) := w(t) - (grad(t) + momentum * grad(t-1)) decrease_const : float (default: 0.0) Decrease constant. Shrinks the learning rate after each epoch via eta / (1 + epoch*decrease_const) minibatches : int (default: 1) Divide the training data into k minibatches for accelerated stochastic gradient descent learning. Gradient Descent Learning if minibatches = 1 Stochastic Gradient Descent learning if minibatches = len(y) Minibatch learning if minibatches > 1 random_seed : int (default: None) Set random state for shuffling and initializing the weights. print_progress : int (default: 0) Prints progress in fitting to stderr. 0: No output 1: Epochs elapsed and cost 2: 1 plus time elapsed 3: 2 plus estimated time until completion Attributes w_ : 2d-array, shape=[n_features, n_classes] Weights after fitting. b_ : 1D-array, shape=[n_classes] Bias units after fitting. cost_ : list List of floats; the mean categorical cross entropy cost after each epoch. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/classifier/MultiLayerPerceptron/","title":"API"},{"location":"user_guide/classifier/MultiLayerPerceptron/#methods","text":"fit(X, y, init_params=True) Learn model from training data. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values. init_params : bool (default: True) Re-initializes model parameters prior to fitting. Set False to continue training with weights from a previous model fitting. Returns self : object predict(X) Predict targets from X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns target_values : array-like, shape = [n_samples] Predicted target values. predict_proba(X) Predict class probabilities of X from the net input. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns Class probabilties : array-like, shape= [n_samples, n_classes] score(X, y) Compute the prediction accuracy Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values (true class labels). Returns acc : float The prediction accuracy as a float between 0.0 and 1.0 (perfect score). ython","title":"Methods"},{"location":"user_guide/classifier/OneRClassifier/","text":"OneRClassifier: One Rule (OneR) method for classfication And implementation of the One Rule (OneR) method for classfication. from mlxtend.classifier import OneRClassifier Overview \"OneR\" stands for One Rule (by Robert Holte [1]), which is a classic algorithm for supervised learning. Note that this algorithm is not known for its good prediction performance; thus, it is rather recommended for teaching purposes and for lower-bound performance baselines in real-world applications. The name \"OneRule\" can be a bit misleading, because it is technically about \"one feature\" and not about \"one rule.\" I.e., OneR returns a feature for which one or more decision rules are defined. Essentially, as a simple classifier, it finds exactly one feature (and one or more feature values for that feature) to classify data instances. The basic procedure is as follows: For each feature among all features (columns) in the dataset: For each feature value for the given feature: Obtain the training examples with that feature value. Obtain the class labels (and class label counts) corresponding to the training examples identified in the previous step. Regard the class label with the highest frequency (count) as the majority class. Record the number of errors as the number of training examples that have the given feature value but are not the majority class. Compute the error of the feature by summing the errors for all possible feature values for that feature. Return the best feature, which is defined as the feature with the lowest error. Please note that the OneR algorithm assumes categorical (or discretized) feature values. A nice explanation and OneR classifier can be found in the Interpretable Machine Learning online chapter \"4.5.1 Learn Rules from a Single Feature (OneR)\" (https://christophm.github.io/interpretable-ml-book/rules.html, [2]). References [1] Holte, Robert C. \"Very simple classification rules perform well on most commonly used datasets.\" Machine learning 11.1 (1993): 63-90. [2] Interpretable Machine Learning (2018) by Christoph Molnar: https://christophm.github.io/interpretable-ml-book/rules.html Example 1 -- Demonstrating OneR on a discretized Iris dataset As mentioned in the overview text above, the OneR algorithm expects categorical or discretized features. The OneRClassifier implementation in MLxtend does not modify features in the dataset, and ensuring that the features are categorical is a responsibility of the user. In the following example, we will discretize the Iris dataset. In particular, we will convert the dataset into quartiles. In other words each feature value gets replaced by a categorical value. For sepal width (the first column in Iris), this would be (0, 5.1] => 0 (5.1, 5.8] => 1 (5.8, 6.4] => 2 (6,4, 7.9] => 3 Below are the first 15 rows (flowers) of the original Iris data: from mlxtend.data import iris_data X, y = iris_data() X[:15] array([[5.1, 3.5, 1.4, 0.2], [4.9, 3. , 1.4, 0.2], [4.7, 3.2, 1.3, 0.2], [4.6, 3.1, 1.5, 0.2], [5. , 3.6, 1.4, 0.2], [5.4, 3.9, 1.7, 0.4], [4.6, 3.4, 1.4, 0.3], [5. , 3.4, 1.5, 0.2], [4.4, 2.9, 1.4, 0.2], [4.9, 3.1, 1.5, 0.1], [5.4, 3.7, 1.5, 0.2], [4.8, 3.4, 1.6, 0.2], [4.8, 3. , 1.4, 0.1], [4.3, 3. , 1.1, 0.1], [5.8, 4. , 1.2, 0.2]]) Below is the discretized dataset. Each feature is divided into 4 quartiles. import numpy as np def get_feature_quartiles(X): X_discretized = X.copy() for col in range(X.shape[1]): for q, class_label in zip([1.0, 0.75, 0.5, 0.25], [3, 2, 1, 0]): threshold = np.quantile(X[:, col], q=q) X_discretized[X[:, col] <= threshold, col] = class_label return X_discretized.astype(np.int) Xd = get_feature_quartiles(X) Xd[:15] array([[0, 3, 0, 0], [0, 1, 0, 0], [0, 2, 0, 0], [0, 2, 0, 0], [0, 3, 0, 0], [1, 3, 1, 1], [0, 3, 0, 0], [0, 3, 0, 0], [0, 1, 0, 0], [0, 2, 0, 0], [1, 3, 0, 0], [0, 3, 0, 0], [0, 1, 0, 0], [0, 1, 0, 0], [1, 3, 0, 0]]) Given a dataset with categorical features, we can use the OneR classifier like similar to a scikit-learn estimator for classification. First, let's divide the dataset into training and test data: from sklearn.model_selection import train_test_split Xd_train, Xd_test, y_train, y_test = train_test_split(Xd, y, random_state=0, stratify=y) Next, we can train a OneRClassifier model on the training set using the fit method: from mlxtend.classifier import OneRClassifier oner = OneRClassifier() oner.fit(Xd_train, y_train); The column index of the selected feature is accessible via the feature_idx_ attribute after model fitting: oner.feature_idx_ 2 There is also a prediction_dict_ available after model fitting. It lists the total error for the selected feature (i.e., the feature listed under feature_idx_ ). Also it provides the classification rules: oner.prediction_dict_ {'total error': 16, 'rules (value: class)': {0: 0, 1: 1, 2: 1, 3: 2}} I.e., 'rules (value: class)': {0: 0, 1: 1, 2: 1, 3: 2} means that there are 3 rules for the selected feature (petal length): - if value is 0, classify as 0 (Iris-setosa) - if value is 1, classify as 1 (Iris-versicolor) - if value is 2, classify as 1 (Iris-versicolor) - if value is 3, classify as 2 (Iris-virginica) After model fitting we can use the oner object for making predictions: oner.predict(Xd_train) array([1, 0, 0, 1, 1, 1, 1, 0, 1, 0, 0, 1, 0, 1, 0, 1, 1, 1, 1, 2, 2, 1, 0, 1, 2, 1, 2, 1, 1, 2, 1, 2, 1, 0, 0, 1, 2, 1, 1, 2, 2, 1, 0, 1, 1, 1, 2, 0, 1, 2, 1, 2, 1, 0, 0, 0, 0, 1, 0, 1, 1, 1, 2, 0, 1, 1, 0, 1, 2, 1, 2, 0, 1, 2, 1, 1, 2, 0, 1, 0, 0, 1, 1, 2, 0, 0, 0, 1, 0, 1, 2, 2, 2, 0, 1, 0, 2, 0, 1, 1, 1, 1, 0, 2, 2, 0, 1, 1, 0, 2, 1, 2]) y_pred = oner.predict(Xd_train) train_acc = np.mean(y_pred == y_train) print(f'Training accuracy {train_acc*100:.2f}%') Training accuracy 85.71% y_pred = oner.predict(Xd_test) test_acc = np.mean(y_pred == y_test) print(f'Test accuracy {test_acc*100:.2f}%') Test accuracy 84.21% Instead of computing the prediction accuracy manually as shown above, we can also use the score method: test_acc = oner.score(Xd_test, y_test) print(f'Test accuracy {test_acc*100:.2f}%') Test accuracy 84.21% API OneRClassifier(resolve_ties='first') OneR (One Rule) Classifier. Parameters resolve_ties : str (default: 'first') Option for how to resolve ties if two or more features have the same error. Options are - 'first' (default): chooses first feature in the list, i.e., feature with the lower column index. - 'chi-squared': performs a chi-squared test for each feature against the target and selects the feature with the lowest p-value. Attributes self.classes_labels_ : array-like, shape = [n_labels] Array containing the unique class labels found in the training set. self.feature_idx_ : int The index of the rules' feature based on the column in the training set. self.p_value_ : float The p value for a given feature. Only available after calling fit when the OneR attribute resolve_ties = 'chi-squared' is set. self.prediction_dict_ : dict Dictionary containing information about the feature's (self.feature_idx_) rules and total error. E.g., {'total error': 37, 'rules (value: class)': {0: 0, 1: 2}} means the total error is 37, and the rules are \"if feature value == 0 classify as 0\" and \"if feature value == 1 classify as 2\". (And classify as class 1 otherwise.) For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/classifier/OneRClassifier/ Methods fit(X, y) Learn rule from training data. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values. Returns self : object get_params(deep=True) Get parameters for this estimator. Parameters deep : bool, default=True If True, will return the parameters for this estimator and contained subobjects that are estimators. Returns params : mapping of string to any Parameter names mapped to their values. predict(X) Predict class labels for X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns maj : array-like, shape = [n_samples] Predicted class labels. score(X, y, sample_weight=None) Return the mean accuracy on the given test data and labels. In multi-label classification, this is the subset accuracy which is a harsh metric since you require for each sample that each label set be correctly predicted. Parameters X : array-like of shape (n_samples, n_features) Test samples. y : array-like of shape (n_samples,) or (n_samples, n_outputs) True labels for X. sample_weight : array-like of shape (n_samples,), default=None Sample weights. Returns score : float Mean accuracy of self.predict(X) wrt. y. set_params( params) Set the parameters of this estimator. The method works on simple estimators as well as on nested objects (such as pipelines). The latter have parameters of the form <component>__<parameter> so that it's possible to update each component of a nested object. Parameters **params : dict Estimator parameters. Returns self : object Estimator instance. ython","title":"OneRClassifier: One Rule (OneR) method for classfication"},{"location":"user_guide/classifier/OneRClassifier/#onerclassifier-one-rule-oner-method-for-classfication","text":"And implementation of the One Rule (OneR) method for classfication. from mlxtend.classifier import OneRClassifier","title":"OneRClassifier: One Rule (OneR) method for classfication"},{"location":"user_guide/classifier/OneRClassifier/#overview","text":"\"OneR\" stands for One Rule (by Robert Holte [1]), which is a classic algorithm for supervised learning. Note that this algorithm is not known for its good prediction performance; thus, it is rather recommended for teaching purposes and for lower-bound performance baselines in real-world applications. The name \"OneRule\" can be a bit misleading, because it is technically about \"one feature\" and not about \"one rule.\" I.e., OneR returns a feature for which one or more decision rules are defined. Essentially, as a simple classifier, it finds exactly one feature (and one or more feature values for that feature) to classify data instances. The basic procedure is as follows: For each feature among all features (columns) in the dataset: For each feature value for the given feature: Obtain the training examples with that feature value. Obtain the class labels (and class label counts) corresponding to the training examples identified in the previous step. Regard the class label with the highest frequency (count) as the majority class. Record the number of errors as the number of training examples that have the given feature value but are not the majority class. Compute the error of the feature by summing the errors for all possible feature values for that feature. Return the best feature, which is defined as the feature with the lowest error. Please note that the OneR algorithm assumes categorical (or discretized) feature values. A nice explanation and OneR classifier can be found in the Interpretable Machine Learning online chapter \"4.5.1 Learn Rules from a Single Feature (OneR)\" (https://christophm.github.io/interpretable-ml-book/rules.html, [2]).","title":"Overview"},{"location":"user_guide/classifier/OneRClassifier/#references","text":"[1] Holte, Robert C. \"Very simple classification rules perform well on most commonly used datasets.\" Machine learning 11.1 (1993): 63-90. [2] Interpretable Machine Learning (2018) by Christoph Molnar: https://christophm.github.io/interpretable-ml-book/rules.html","title":"References"},{"location":"user_guide/classifier/OneRClassifier/#example-1-demonstrating-oner-on-a-discretized-iris-dataset","text":"As mentioned in the overview text above, the OneR algorithm expects categorical or discretized features. The OneRClassifier implementation in MLxtend does not modify features in the dataset, and ensuring that the features are categorical is a responsibility of the user. In the following example, we will discretize the Iris dataset. In particular, we will convert the dataset into quartiles. In other words each feature value gets replaced by a categorical value. For sepal width (the first column in Iris), this would be (0, 5.1] => 0 (5.1, 5.8] => 1 (5.8, 6.4] => 2 (6,4, 7.9] => 3 Below are the first 15 rows (flowers) of the original Iris data: from mlxtend.data import iris_data X, y = iris_data() X[:15] array([[5.1, 3.5, 1.4, 0.2], [4.9, 3. , 1.4, 0.2], [4.7, 3.2, 1.3, 0.2], [4.6, 3.1, 1.5, 0.2], [5. , 3.6, 1.4, 0.2], [5.4, 3.9, 1.7, 0.4], [4.6, 3.4, 1.4, 0.3], [5. , 3.4, 1.5, 0.2], [4.4, 2.9, 1.4, 0.2], [4.9, 3.1, 1.5, 0.1], [5.4, 3.7, 1.5, 0.2], [4.8, 3.4, 1.6, 0.2], [4.8, 3. , 1.4, 0.1], [4.3, 3. , 1.1, 0.1], [5.8, 4. , 1.2, 0.2]]) Below is the discretized dataset. Each feature is divided into 4 quartiles. import numpy as np def get_feature_quartiles(X): X_discretized = X.copy() for col in range(X.shape[1]): for q, class_label in zip([1.0, 0.75, 0.5, 0.25], [3, 2, 1, 0]): threshold = np.quantile(X[:, col], q=q) X_discretized[X[:, col] <= threshold, col] = class_label return X_discretized.astype(np.int) Xd = get_feature_quartiles(X) Xd[:15] array([[0, 3, 0, 0], [0, 1, 0, 0], [0, 2, 0, 0], [0, 2, 0, 0], [0, 3, 0, 0], [1, 3, 1, 1], [0, 3, 0, 0], [0, 3, 0, 0], [0, 1, 0, 0], [0, 2, 0, 0], [1, 3, 0, 0], [0, 3, 0, 0], [0, 1, 0, 0], [0, 1, 0, 0], [1, 3, 0, 0]]) Given a dataset with categorical features, we can use the OneR classifier like similar to a scikit-learn estimator for classification. First, let's divide the dataset into training and test data: from sklearn.model_selection import train_test_split Xd_train, Xd_test, y_train, y_test = train_test_split(Xd, y, random_state=0, stratify=y) Next, we can train a OneRClassifier model on the training set using the fit method: from mlxtend.classifier import OneRClassifier oner = OneRClassifier() oner.fit(Xd_train, y_train); The column index of the selected feature is accessible via the feature_idx_ attribute after model fitting: oner.feature_idx_ 2 There is also a prediction_dict_ available after model fitting. It lists the total error for the selected feature (i.e., the feature listed under feature_idx_ ). Also it provides the classification rules: oner.prediction_dict_ {'total error': 16, 'rules (value: class)': {0: 0, 1: 1, 2: 1, 3: 2}} I.e., 'rules (value: class)': {0: 0, 1: 1, 2: 1, 3: 2} means that there are 3 rules for the selected feature (petal length): - if value is 0, classify as 0 (Iris-setosa) - if value is 1, classify as 1 (Iris-versicolor) - if value is 2, classify as 1 (Iris-versicolor) - if value is 3, classify as 2 (Iris-virginica) After model fitting we can use the oner object for making predictions: oner.predict(Xd_train) array([1, 0, 0, 1, 1, 1, 1, 0, 1, 0, 0, 1, 0, 1, 0, 1, 1, 1, 1, 2, 2, 1, 0, 1, 2, 1, 2, 1, 1, 2, 1, 2, 1, 0, 0, 1, 2, 1, 1, 2, 2, 1, 0, 1, 1, 1, 2, 0, 1, 2, 1, 2, 1, 0, 0, 0, 0, 1, 0, 1, 1, 1, 2, 0, 1, 1, 0, 1, 2, 1, 2, 0, 1, 2, 1, 1, 2, 0, 1, 0, 0, 1, 1, 2, 0, 0, 0, 1, 0, 1, 2, 2, 2, 0, 1, 0, 2, 0, 1, 1, 1, 1, 0, 2, 2, 0, 1, 1, 0, 2, 1, 2]) y_pred = oner.predict(Xd_train) train_acc = np.mean(y_pred == y_train) print(f'Training accuracy {train_acc*100:.2f}%') Training accuracy 85.71% y_pred = oner.predict(Xd_test) test_acc = np.mean(y_pred == y_test) print(f'Test accuracy {test_acc*100:.2f}%') Test accuracy 84.21% Instead of computing the prediction accuracy manually as shown above, we can also use the score method: test_acc = oner.score(Xd_test, y_test) print(f'Test accuracy {test_acc*100:.2f}%') Test accuracy 84.21%","title":"Example 1 -- Demonstrating OneR on a discretized Iris dataset"},{"location":"user_guide/classifier/OneRClassifier/#api","text":"OneRClassifier(resolve_ties='first') OneR (One Rule) Classifier. Parameters resolve_ties : str (default: 'first') Option for how to resolve ties if two or more features have the same error. Options are - 'first' (default): chooses first feature in the list, i.e., feature with the lower column index. - 'chi-squared': performs a chi-squared test for each feature against the target and selects the feature with the lowest p-value. Attributes self.classes_labels_ : array-like, shape = [n_labels] Array containing the unique class labels found in the training set. self.feature_idx_ : int The index of the rules' feature based on the column in the training set. self.p_value_ : float The p value for a given feature. Only available after calling fit when the OneR attribute resolve_ties = 'chi-squared' is set. self.prediction_dict_ : dict Dictionary containing information about the feature's (self.feature_idx_) rules and total error. E.g., {'total error': 37, 'rules (value: class)': {0: 0, 1: 2}} means the total error is 37, and the rules are \"if feature value == 0 classify as 0\" and \"if feature value == 1 classify as 2\". (And classify as class 1 otherwise.) For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/classifier/OneRClassifier/","title":"API"},{"location":"user_guide/classifier/OneRClassifier/#methods","text":"fit(X, y) Learn rule from training data. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values. Returns self : object get_params(deep=True) Get parameters for this estimator. Parameters deep : bool, default=True If True, will return the parameters for this estimator and contained subobjects that are estimators. Returns params : mapping of string to any Parameter names mapped to their values. predict(X) Predict class labels for X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns maj : array-like, shape = [n_samples] Predicted class labels. score(X, y, sample_weight=None) Return the mean accuracy on the given test data and labels. In multi-label classification, this is the subset accuracy which is a harsh metric since you require for each sample that each label set be correctly predicted. Parameters X : array-like of shape (n_samples, n_features) Test samples. y : array-like of shape (n_samples,) or (n_samples, n_outputs) True labels for X. sample_weight : array-like of shape (n_samples,), default=None Sample weights. Returns score : float Mean accuracy of self.predict(X) wrt. y. set_params( params) Set the parameters of this estimator. The method works on simple estimators as well as on nested objects (such as pipelines). The latter have parameters of the form <component>__<parameter> so that it's possible to update each component of a nested object. Parameters **params : dict Estimator parameters. Returns self : object Estimator instance. ython","title":"Methods"},{"location":"user_guide/classifier/Perceptron/","text":"Perceptron: A simple binary classifier Implementation of a Perceptron learning algorithm for classification. from mlxtend.classifier import Perceptron Overview The idea behind this \"thresholded\" perceptron was to mimic how a single neuron in the brain works: It either \"fires\" or not. A perceptron receives multiple input signals, and if the sum of the input signals exceed a certain threshold it either returns a signal or remains \"silent\" otherwise. What made this a \"machine learning\" algorithm was Frank Rosenblatt's idea of the perceptron learning rule: The perceptron algorithm is about learning the weights for the input signals in order to draw linear decision boundary that allows us to discriminate between the two linearly separable classes +1 and -1. Basic Notation Before we dive deeper into the algorithm(s) for learning the weights of the perceptron classifier, let us take a brief look at the basic notation. In the following sections, we will label the positive and negative class in our binary classification setting as \"1\" and \"-1\", respectively. Next, we define an activation function g(\\mathbf{z}) that takes a linear combination of the input values \\mathbf{x} and weights \\mathbf{w} as input ( \\mathbf{z} = w_1x_{1} + \\dots + w_mx_{m} ), and if g(\\mathbf{z}) is greater than a defined threshold \\theta we predict 1 and -1 otherwise; in this case, this activation function g is a simple \"unit step function,\" which is sometimes also called \"Heaviside step function.\" $$ g(z) = \\begin{cases} 1 & \\text{if $z \\ge \\theta$}\\\\ -1 & \\text{otherwise}. \\end{cases} $$ where z = w_1x_{1} + \\dots + w_mx_{m} = \\sum_{j=1}^{m} x_{j}w_{j} \\\\ = \\mathbf{w}^T\\mathbf{x} \\mathbf{w} is the feature vector, and \\mathbf{x} is an m -dimensional sample from the training dataset: \\mathbf{w} = \\begin{bmatrix} w_{1} \\\\ \\vdots \\\\ w_{m} \\end{bmatrix} \\quad \\mathbf{x} = \\begin{bmatrix} x_{1} \\\\ \\vdots \\\\ x_{m} \\end{bmatrix} In order to simplify the notation, we bring \\theta to the left side of the equation and define w_0 = -\\theta \\text{ and } x_0=1 so that $$ g({z}) = \\begin{cases} 1 & \\text{if $z \\ge 0$}\\\\ -1 & \\text{otherwise}. \\end{cases} $$ and z = w_0x_{0} + w_1x_{1} + \\dots + w_mx_{m} = \\sum_{j=0}^{m} x_{j}w_{j} \\\\ = \\mathbf{w}^T\\mathbf{x}. Perceptron Rule Rosenblatt's initial perceptron rule is fairly simple and can be summarized by the following steps: Initialize the weights to 0 or small random numbers. For each training sample \\mathbf{x^{(i)}} : Calculate the output value. Update the weights. The output value is the class label predicted by the unit step function that we defined earlier (output =g(\\mathbf{z}) ) and the weight update can be written more formally as w_j := w_j + \\Delta w_j . The value for updating the weights at each increment is calculated by the learning rule \\Delta w_j = \\eta \\; (\\text{target}^{(i)} - \\text{output}^{(i)})\\;x^{(i)}_{j} where \\eta is the learning rate (a constant between 0.0 and 1.0), \"target\" is the true class label, and the \"output\" is the predicted class label. aIt is important to note that all weights in the weight vector are being updated simultaneously. Concretely, for a 2-dimensional dataset, we would write the update as: \\Delta w_0 = \\eta(\\text{target}^{(i)} - \\text{output}^{(i)}) \\Delta w_1 = \\eta(\\text{target}^{(i)} - \\text{output}^{(i)})\\;x^{(i)}_{1} \\Delta w_2 = \\eta(\\text{target}^{(i)} - \\text{output}^{(i)})\\;x^{(i)}_{2} Before we implement the perceptron rule in Python, let us make a simple thought experiment to illustrate how beautifully simple this learning rule really is. In the two scenarios where the perceptron predicts the class label correctly, the weights remain unchanged: \\Delta w_j = \\eta(-1^{(i)} - -1^{(i)})\\;x^{(i)}_{j} = 0 \\Delta w_j = \\eta(1^{(i)} - 1^{(i)})\\;x^{(i)}_{j} = 0 However, in case of a wrong prediction, the weights are being \"pushed\" towards the direction of the positive or negative target class, respectively: \\Delta w_j = \\eta(1^{(i)} - -1^{(i)})\\;x^{(i)}_{j} = \\eta(2)\\;x^{(i)}_{j} \\Delta w_j = \\eta(-1^{(i)} - 1^{(i)})\\;x^{(i)}_{j} = \\eta(-2)\\;x^{(i)}_{j} It is important to note that the convergence of the perceptron is only guaranteed if the two classes are linearly separable. If the two classes can't be separated by a linear decision boundary, we can set a maximum number of passes over the training dataset (\"epochs\") and/or a threshold for the number of tolerated misclassifications. References F. Rosenblatt. The perceptron, a perceiving and recognizing automaton Project Para. Cornell Aeronautical Laboratory, 1957. Example 1 - Classification of Iris Flowers from mlxtend.data import iris_data from mlxtend.plotting import plot_decision_regions from mlxtend.classifier import Perceptron import matplotlib.pyplot as plt # Loading Data X, y = iris_data() X = X[:, [0, 3]] # sepal length and petal width X = X[0:100] # class 0 and class 1 y = y[0:100] # class 0 and class 1 # standardize X[:,0] = (X[:,0] - X[:,0].mean()) / X[:,0].std() X[:,1] = (X[:,1] - X[:,1].mean()) / X[:,1].std() # Rosenblatt Perceptron ppn = Perceptron(epochs=5, eta=0.05, random_seed=0, print_progress=3) ppn.fit(X, y) plot_decision_regions(X, y, clf=ppn) plt.title('Perceptron - Rosenblatt Perceptron Rule') plt.show() print('Bias & Weights: %s' % ppn.w_) plt.plot(range(len(ppn.cost_)), ppn.cost_) plt.xlabel('Iterations') plt.ylabel('Missclassifications') plt.show() Iteration: 5/5 | Elapsed: 00:00:00 | ETA: 00:00:00 Bias & Weights: [[-0.04500809] [ 0.11048855]] API Perceptron(eta=0.1, epochs=50, random_seed=None, print_progress=0) Perceptron classifier. Note that this implementation of the Perceptron expects binary class labels in {0, 1}. Parameters eta : float (default: 0.1) Learning rate (between 0.0 and 1.0) epochs : int (default: 50) Number of passes over the training dataset. Prior to each epoch, the dataset is shuffled to prevent cycles. random_seed : int Random state for initializing random weights and shuffling. print_progress : int (default: 0) Prints progress in fitting to stderr. 0: No output 1: Epochs elapsed and cost 2: 1 plus time elapsed 3: 2 plus estimated time until completion Attributes w_ : 2d-array, shape={n_features, 1} Model weights after fitting. b_ : 1d-array, shape={1,} Bias unit after fitting. cost_ : list Number of misclassifications in every epoch. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/classifier/Perceptron/ Methods fit(X, y, init_params=True) Learn model from training data. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values. init_params : bool (default: True) Re-initializes model parameters prior to fitting. Set False to continue training with weights from a previous model fitting. Returns self : object predict(X) Predict targets from X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns target_values : array-like, shape = [n_samples] Predicted target values. score(X, y) Compute the prediction accuracy Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values (true class labels). Returns acc : float The prediction accuracy as a float between 0.0 and 1.0 (perfect score). ython","title":"Perceptron: A simple binary classifier"},{"location":"user_guide/classifier/Perceptron/#perceptron-a-simple-binary-classifier","text":"Implementation of a Perceptron learning algorithm for classification. from mlxtend.classifier import Perceptron","title":"Perceptron: A simple binary classifier"},{"location":"user_guide/classifier/Perceptron/#overview","text":"The idea behind this \"thresholded\" perceptron was to mimic how a single neuron in the brain works: It either \"fires\" or not. A perceptron receives multiple input signals, and if the sum of the input signals exceed a certain threshold it either returns a signal or remains \"silent\" otherwise. What made this a \"machine learning\" algorithm was Frank Rosenblatt's idea of the perceptron learning rule: The perceptron algorithm is about learning the weights for the input signals in order to draw linear decision boundary that allows us to discriminate between the two linearly separable classes +1 and -1.","title":"Overview"},{"location":"user_guide/classifier/Perceptron/#basic-notation","text":"Before we dive deeper into the algorithm(s) for learning the weights of the perceptron classifier, let us take a brief look at the basic notation. In the following sections, we will label the positive and negative class in our binary classification setting as \"1\" and \"-1\", respectively. Next, we define an activation function g(\\mathbf{z}) that takes a linear combination of the input values \\mathbf{x} and weights \\mathbf{w} as input ( \\mathbf{z} = w_1x_{1} + \\dots + w_mx_{m} ), and if g(\\mathbf{z}) is greater than a defined threshold \\theta we predict 1 and -1 otherwise; in this case, this activation function g is a simple \"unit step function,\" which is sometimes also called \"Heaviside step function.\" $$ g(z) = \\begin{cases} 1 & \\text{if $z \\ge \\theta$}\\\\ -1 & \\text{otherwise}. \\end{cases} $$ where z = w_1x_{1} + \\dots + w_mx_{m} = \\sum_{j=1}^{m} x_{j}w_{j} \\\\ = \\mathbf{w}^T\\mathbf{x} \\mathbf{w} is the feature vector, and \\mathbf{x} is an m -dimensional sample from the training dataset: \\mathbf{w} = \\begin{bmatrix} w_{1} \\\\ \\vdots \\\\ w_{m} \\end{bmatrix} \\quad \\mathbf{x} = \\begin{bmatrix} x_{1} \\\\ \\vdots \\\\ x_{m} \\end{bmatrix} In order to simplify the notation, we bring \\theta to the left side of the equation and define w_0 = -\\theta \\text{ and } x_0=1 so that $$ g({z}) = \\begin{cases} 1 & \\text{if $z \\ge 0$}\\\\ -1 & \\text{otherwise}. \\end{cases} $$ and z = w_0x_{0} + w_1x_{1} + \\dots + w_mx_{m} = \\sum_{j=0}^{m} x_{j}w_{j} \\\\ = \\mathbf{w}^T\\mathbf{x}.","title":"Basic Notation"},{"location":"user_guide/classifier/Perceptron/#perceptron-rule","text":"Rosenblatt's initial perceptron rule is fairly simple and can be summarized by the following steps: Initialize the weights to 0 or small random numbers. For each training sample \\mathbf{x^{(i)}} : Calculate the output value. Update the weights. The output value is the class label predicted by the unit step function that we defined earlier (output =g(\\mathbf{z}) ) and the weight update can be written more formally as w_j := w_j + \\Delta w_j . The value for updating the weights at each increment is calculated by the learning rule \\Delta w_j = \\eta \\; (\\text{target}^{(i)} - \\text{output}^{(i)})\\;x^{(i)}_{j} where \\eta is the learning rate (a constant between 0.0 and 1.0), \"target\" is the true class label, and the \"output\" is the predicted class label. aIt is important to note that all weights in the weight vector are being updated simultaneously. Concretely, for a 2-dimensional dataset, we would write the update as: \\Delta w_0 = \\eta(\\text{target}^{(i)} - \\text{output}^{(i)}) \\Delta w_1 = \\eta(\\text{target}^{(i)} - \\text{output}^{(i)})\\;x^{(i)}_{1} \\Delta w_2 = \\eta(\\text{target}^{(i)} - \\text{output}^{(i)})\\;x^{(i)}_{2} Before we implement the perceptron rule in Python, let us make a simple thought experiment to illustrate how beautifully simple this learning rule really is. In the two scenarios where the perceptron predicts the class label correctly, the weights remain unchanged: \\Delta w_j = \\eta(-1^{(i)} - -1^{(i)})\\;x^{(i)}_{j} = 0 \\Delta w_j = \\eta(1^{(i)} - 1^{(i)})\\;x^{(i)}_{j} = 0 However, in case of a wrong prediction, the weights are being \"pushed\" towards the direction of the positive or negative target class, respectively: \\Delta w_j = \\eta(1^{(i)} - -1^{(i)})\\;x^{(i)}_{j} = \\eta(2)\\;x^{(i)}_{j} \\Delta w_j = \\eta(-1^{(i)} - 1^{(i)})\\;x^{(i)}_{j} = \\eta(-2)\\;x^{(i)}_{j} It is important to note that the convergence of the perceptron is only guaranteed if the two classes are linearly separable. If the two classes can't be separated by a linear decision boundary, we can set a maximum number of passes over the training dataset (\"epochs\") and/or a threshold for the number of tolerated misclassifications.","title":"Perceptron Rule"},{"location":"user_guide/classifier/Perceptron/#references","text":"F. Rosenblatt. The perceptron, a perceiving and recognizing automaton Project Para. Cornell Aeronautical Laboratory, 1957.","title":"References"},{"location":"user_guide/classifier/Perceptron/#example-1-classification-of-iris-flowers","text":"from mlxtend.data import iris_data from mlxtend.plotting import plot_decision_regions from mlxtend.classifier import Perceptron import matplotlib.pyplot as plt # Loading Data X, y = iris_data() X = X[:, [0, 3]] # sepal length and petal width X = X[0:100] # class 0 and class 1 y = y[0:100] # class 0 and class 1 # standardize X[:,0] = (X[:,0] - X[:,0].mean()) / X[:,0].std() X[:,1] = (X[:,1] - X[:,1].mean()) / X[:,1].std() # Rosenblatt Perceptron ppn = Perceptron(epochs=5, eta=0.05, random_seed=0, print_progress=3) ppn.fit(X, y) plot_decision_regions(X, y, clf=ppn) plt.title('Perceptron - Rosenblatt Perceptron Rule') plt.show() print('Bias & Weights: %s' % ppn.w_) plt.plot(range(len(ppn.cost_)), ppn.cost_) plt.xlabel('Iterations') plt.ylabel('Missclassifications') plt.show() Iteration: 5/5 | Elapsed: 00:00:00 | ETA: 00:00:00 Bias & Weights: [[-0.04500809] [ 0.11048855]]","title":"Example 1 - Classification of Iris Flowers"},{"location":"user_guide/classifier/Perceptron/#api","text":"Perceptron(eta=0.1, epochs=50, random_seed=None, print_progress=0) Perceptron classifier. Note that this implementation of the Perceptron expects binary class labels in {0, 1}. Parameters eta : float (default: 0.1) Learning rate (between 0.0 and 1.0) epochs : int (default: 50) Number of passes over the training dataset. Prior to each epoch, the dataset is shuffled to prevent cycles. random_seed : int Random state for initializing random weights and shuffling. print_progress : int (default: 0) Prints progress in fitting to stderr. 0: No output 1: Epochs elapsed and cost 2: 1 plus time elapsed 3: 2 plus estimated time until completion Attributes w_ : 2d-array, shape={n_features, 1} Model weights after fitting. b_ : 1d-array, shape={1,} Bias unit after fitting. cost_ : list Number of misclassifications in every epoch. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/classifier/Perceptron/","title":"API"},{"location":"user_guide/classifier/Perceptron/#methods","text":"fit(X, y, init_params=True) Learn model from training data. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values. init_params : bool (default: True) Re-initializes model parameters prior to fitting. Set False to continue training with weights from a previous model fitting. Returns self : object predict(X) Predict targets from X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns target_values : array-like, shape = [n_samples] Predicted target values. score(X, y) Compute the prediction accuracy Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values (true class labels). Returns acc : float The prediction accuracy as a float between 0.0 and 1.0 (perfect score). ython","title":"Methods"},{"location":"user_guide/classifier/SoftmaxRegression/","text":"SoftmaxRegression: Multiclass version of logistic regression A logistic regression class for multi-class classification tasks. from mlxtend.classifier import SoftmaxRegression Overview Softmax Regression (synonyms: Multinomial Logistic , Maximum Entropy Classifier , or just Multi-class Logistic Regression ) is a generalization of logistic regression that we can use for multi-class classification (under the assumption that the classes are mutually exclusive). In contrast, we use the (standard) Logistic Regression model in binary classification tasks. Below is a schematic of a Logistic Regression model, for more details, please see the LogisticRegression manual . In Softmax Regression (SMR), we replace the sigmoid logistic function by the so-called softmax function \\phi_{softmax}(\\cdot) . P(y=j \\mid z^{(i)}) = \\phi_{softmax}(z^{(i)}) = \\frac{e^{z^{(i)}}}{\\sum_{j=0}^{k} e^{z_{k}^{(i)}}}, where we define the net input z as z = w_1x_1 + ... + w_mx_m + b= \\sum_{l=1}^{m} w_l x_l + b= \\mathbf{w}^T\\mathbf{x} + b. ( w is the weight vector, \\mathbf{x} is the feature vector of 1 training sample, and b is the bias unit.) Now, this softmax function computes the probability that this training sample \\mathbf{x}^{(i)} belongs to class j given the weight and net input z^{(i)} . So, we compute the probability p(y = j \\mid \\mathbf{x^{(i)}; w}_j) for each class label in j = 1, \\ldots, k. . Note the normalization term in the denominator which causes these class probabilities to sum up to one. To illustrate the concept of softmax, let us walk through a concrete example. Let's assume we have a training set consisting of 4 samples from 3 different classes (0, 1, and 2) x_0 \\rightarrow \\text{class }0 x_1 \\rightarrow \\text{class }1 x_2 \\rightarrow \\text{class }2 x_3 \\rightarrow \\text{class }2 import numpy as np y = np.array([0, 1, 2, 2]) First, we want to encode the class labels into a format that we can more easily work with; we apply one-hot encoding: y_enc = (np.arange(np.max(y) + 1) == y[:, None]).astype(float) print('one-hot encoding:\\n', y_enc) one-hot encoding: [[ 1. 0. 0.] [ 0. 1. 0.] [ 0. 0. 1.] [ 0. 0. 1.]] A sample that belongs to class 0 (the first row) has a 1 in the first cell, a sample that belongs to class 2 has a 1 in the second cell of its row, and so forth. Next, let us define the feature matrix of our 4 training samples. Here, we assume that our dataset consists of 2 features; thus, we create a 4x2 dimensional matrix of our samples and features. Similarly, we create a 2x3 dimensional weight matrix (one row per feature and one column for each class). X = np.array([[0.1, 0.5], [1.1, 2.3], [-1.1, -2.3], [-1.5, -2.5]]) W = np.array([[0.1, 0.2, 0.3], [0.1, 0.2, 0.3]]) bias = np.array([0.01, 0.1, 0.1]) print('Inputs X:\\n', X) print('\\nWeights W:\\n', W) print('\\nbias:\\n', bias) Inputs X: [[ 0.1 0.5] [ 1.1 2.3] [-1.1 -2.3] [-1.5 -2.5]] Weights W: [[ 0.1 0.2 0.3] [ 0.1 0.2 0.3]] bias: [ 0.01 0.1 0.1 ] To compute the net input, we multiply the 4x2 matrix feature matrix X with the 2x3 (n_features x n_classes) weight matrix W , which yields a 4x3 output matrix (n_samples x n_classes) to which we then add the bias unit: \\mathbf{Z} = \\mathbf{X}\\mathbf{W} + \\mathbf{b}. X = np.array([[0.1, 0.5], [1.1, 2.3], [-1.1, -2.3], [-1.5, -2.5]]) W = np.array([[0.1, 0.2, 0.3], [0.1, 0.2, 0.3]]) bias = np.array([0.01, 0.1, 0.1]) print('Inputs X:\\n', X) print('\\nWeights W:\\n', W) print('\\nbias:\\n', bias) Inputs X: [[ 0.1 0.5] [ 1.1 2.3] [-1.1 -2.3] [-1.5 -2.5]] Weights W: [[ 0.1 0.2 0.3] [ 0.1 0.2 0.3]] bias: [ 0.01 0.1 0.1 ] def net_input(X, W, b): return (X.dot(W) + b) net_in = net_input(X, W, bias) print('net input:\\n', net_in) net input: [[ 0.07 0.22 0.28] [ 0.35 0.78 1.12] [-0.33 -0.58 -0.92] [-0.39 -0.7 -1.1 ]] Now, it's time to compute the softmax activation that we discussed earlier: P(y=j \\mid z^{(i)}) = \\phi_{softmax}(z^{(i)}) = \\frac{e^{z^{(i)}}}{\\sum_{j=0}^{k} e^{z_{k}^{(i)}}}. def softmax(z): return (np.exp(z.T) / np.sum(np.exp(z), axis=1)).T smax = softmax(net_in) print('softmax:\\n', smax) softmax: [[ 0.29450637 0.34216758 0.36332605] [ 0.21290077 0.32728332 0.45981591] [ 0.42860913 0.33380113 0.23758974] [ 0.44941979 0.32962558 0.22095463]] As we can see, the values for each sample (row) nicely sum up to 1 now. E.g., we can say that the first sample [ 0.29450637 0.34216758 0.36332605] has a 29.45% probability to belong to class 0. Now, in order to turn these probabilities back into class labels, we could simply take the argmax-index position of each row: [[ 0.29450637 0.34216758 0.36332605 ] -> 2 [ 0.21290077 0.32728332 0.45981591 ] -> 2 [ 0.42860913 0.33380113 0.23758974] -> 0 [ 0.44941979 0.32962558 0.22095463]] -> 0 def to_classlabel(z): return z.argmax(axis=1) print('predicted class labels: ', to_classlabel(smax)) predicted class labels: [2 2 0 0] As we can see, our predictions are terribly wrong, since the correct class labels are [0, 1, 2, 2] . Now, in order to train our logistic model (e.g., via an optimization algorithm such as gradient descent), we need to define a cost function J(\\cdot) that we want to minimize: J(\\mathbf{W}; \\mathbf{b}) = \\frac{1}{n} \\sum_{i=1}^{n} H(T_i, O_i), which is the average of all cross-entropies over our n training samples. The cross-entropy function is defined as H(T_i, O_i) = -\\sum_m T_i \\cdot log(O_i). Here the T stands for \"target\" (i.e., the true class labels) and the O stands for output -- the computed probability via softmax; not the predicted class label. def cross_entropy(output, y_target): return - np.sum(np.log(output) * (y_target), axis=1) xent = cross_entropy(smax, y_enc) print('Cross Entropy:', xent) Cross Entropy: [ 1.22245465 1.11692907 1.43720989 1.50979788] def cost(output, y_target): return np.mean(cross_entropy(output, y_target)) J_cost = cost(smax, y_enc) print('Cost: ', J_cost) Cost: 1.32159787159 In order to learn our softmax model -- determining the weight coefficients -- via gradient descent, we then need to compute the derivative \\nabla \\mathbf{w}_j \\, J(\\mathbf{W}; \\mathbf{b}). I don't want to walk through the tedious details here, but this cost derivative turns out to be simply: \\nabla \\mathbf{w}_j \\, J(\\mathbf{W}; \\mathbf{b}) = \\frac{1}{n} \\sum^{n}_{i=0} \\big[\\mathbf{x}^{(i)}\\ \\big(O_i - T_i \\big) \\big] We can then use the cost derivate to update the weights in opposite direction of the cost gradient with learning rate \\eta : \\mathbf{w}_j := \\mathbf{w}_j - \\eta \\nabla \\mathbf{w}_j \\, J(\\mathbf{W}; \\mathbf{b}) for each class j \\in \\{0, 1, ..., k\\} (note that \\mathbf{w}_j is the weight vector for the class y=j ), and we update the bias units \\mathbf{b}_j := \\mathbf{b}_j - \\eta \\bigg[ \\frac{1}{n} \\sum^{n}_{i=0} \\big(O_i - T_i \\big) \\bigg]. As a penalty against complexity, an approach to reduce the variance of our model and decrease the degree of overfitting by adding additional bias, we can further add a regularization term such as the L2 term with the regularization parameter \\lambda : L2: \\frac{\\lambda}{2} ||\\mathbf{w}||_{2}^{2} , where ||\\mathbf{w}||_{2}^{2} = \\sum^{m}_{l=0} \\sum^{k}_{j=0} w_{i, j} so that our cost function becomes J(\\mathbf{W}; \\mathbf{b}) = \\frac{1}{n} \\sum_{i=1}^{n} H(T_i, O_i) + \\frac{\\lambda}{2} ||\\mathbf{w}||_{2}^{2} and we define the \"regularized\" weight update as \\mathbf{w}_j := \\mathbf{w}_j - \\eta \\big[\\nabla \\mathbf{w}_j \\, J(\\mathbf{W}) + \\lambda \\mathbf{w}_j \\big]. (Please note that we don't regularize the bias term.) Example 1 - Gradient Descent from mlxtend.data import iris_data from mlxtend.plotting import plot_decision_regions from mlxtend.classifier import SoftmaxRegression import matplotlib.pyplot as plt # Loading Data X, y = iris_data() X = X[:, [0, 3]] # sepal length and petal width # standardize X[:,0] = (X[:,0] - X[:,0].mean()) / X[:,0].std() X[:,1] = (X[:,1] - X[:,1].mean()) / X[:,1].std() lr = SoftmaxRegression(eta=0.01, epochs=500, minibatches=1, random_seed=1, print_progress=3) lr.fit(X, y) plot_decision_regions(X, y, clf=lr) plt.title('Softmax Regression - Gradient Descent') plt.show() plt.plot(range(len(lr.cost_)), lr.cost_) plt.xlabel('Iterations') plt.ylabel('Cost') plt.show() Iteration: 500/500 | Cost 0.06 | Elapsed: 0:00:00 | ETA: 0:00:00 Predicting Class Labels y_pred = lr.predict(X) print('Last 3 Class Labels: %s' % y_pred[-3:]) Last 3 Class Labels: [2 2 2] Predicting Class Probabilities y_pred = lr.predict_proba(X) print('Last 3 Class Labels:\\n %s' % y_pred[-3:]) Last 3 Class Labels: [[ 9.18728149e-09 1.68894679e-02 9.83110523e-01] [ 2.97052325e-11 7.26356627e-04 9.99273643e-01] [ 1.57464093e-06 1.57779528e-01 8.42218897e-01]] Example 2 - Stochastic Gradient Descent from mlxtend.data import iris_data from mlxtend.plotting import plot_decision_regions from mlxtend.classifier import SoftmaxRegression import matplotlib.pyplot as plt # Loading Data X, y = iris_data() X = X[:, [0, 3]] # sepal length and petal width # standardize X[:,0] = (X[:,0] - X[:,0].mean()) / X[:,0].std() X[:,1] = (X[:,1] - X[:,1].mean()) / X[:,1].std() lr = SoftmaxRegression(eta=0.01, epochs=300, minibatches=len(y), random_seed=1) lr.fit(X, y) plot_decision_regions(X, y, clf=lr) plt.title('Softmax Regression - Stochastic Gradient Descent') plt.show() plt.plot(range(len(lr.cost_)), lr.cost_) plt.xlabel('Iterations') plt.ylabel('Cost') plt.show() API SoftmaxRegression(eta=0.01, epochs=50, l2=0.0, minibatches=1, n_classes=None, random_seed=None, print_progress=0) Softmax regression classifier. Parameters eta : float (default: 0.01) Learning rate (between 0.0 and 1.0) epochs : int (default: 50) Passes over the training dataset. Prior to each epoch, the dataset is shuffled if minibatches > 1 to prevent cycles in stochastic gradient descent. l2 : float Regularization parameter for L2 regularization. No regularization if l2=0.0. minibatches : int (default: 1) The number of minibatches for gradient-based optimization. If 1: Gradient Descent learning If len(y): Stochastic Gradient Descent (SGD) online learning If 1 < minibatches < len(y): SGD Minibatch learning n_classes : int (default: None) A positive integer to declare the number of class labels if not all class labels are present in a partial training set. Gets the number of class labels automatically if None. random_seed : int (default: None) Set random state for shuffling and initializing the weights. print_progress : int (default: 0) Prints progress in fitting to stderr. 0: No output 1: Epochs elapsed and cost 2: 1 plus time elapsed 3: 2 plus estimated time until completion Attributes w_ : 2d-array, shape={n_features, 1} Model weights after fitting. b_ : 1d-array, shape={1,} Bias unit after fitting. cost_ : list List of floats, the average cross_entropy for each epoch. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/classifier/SoftmaxRegression/ Methods fit(X, y, init_params=True) Learn model from training data. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values. init_params : bool (default: True) Re-initializes model parameters prior to fitting. Set False to continue training with weights from a previous model fitting. Returns self : object predict(X) Predict targets from X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns target_values : array-like, shape = [n_samples] Predicted target values. predict_proba(X) Predict class probabilities of X from the net input. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns Class probabilties : array-like, shape= [n_samples, n_classes] score(X, y) Compute the prediction accuracy Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values (true class labels). Returns acc : float The prediction accuracy as a float between 0.0 and 1.0 (perfect score). ython","title":"SoftmaxRegression: Multiclass version of logistic regression"},{"location":"user_guide/classifier/SoftmaxRegression/#softmaxregression-multiclass-version-of-logistic-regression","text":"A logistic regression class for multi-class classification tasks. from mlxtend.classifier import SoftmaxRegression","title":"SoftmaxRegression: Multiclass version of logistic regression"},{"location":"user_guide/classifier/SoftmaxRegression/#overview","text":"Softmax Regression (synonyms: Multinomial Logistic , Maximum Entropy Classifier , or just Multi-class Logistic Regression ) is a generalization of logistic regression that we can use for multi-class classification (under the assumption that the classes are mutually exclusive). In contrast, we use the (standard) Logistic Regression model in binary classification tasks. Below is a schematic of a Logistic Regression model, for more details, please see the LogisticRegression manual . In Softmax Regression (SMR), we replace the sigmoid logistic function by the so-called softmax function \\phi_{softmax}(\\cdot) . P(y=j \\mid z^{(i)}) = \\phi_{softmax}(z^{(i)}) = \\frac{e^{z^{(i)}}}{\\sum_{j=0}^{k} e^{z_{k}^{(i)}}}, where we define the net input z as z = w_1x_1 + ... + w_mx_m + b= \\sum_{l=1}^{m} w_l x_l + b= \\mathbf{w}^T\\mathbf{x} + b. ( w is the weight vector, \\mathbf{x} is the feature vector of 1 training sample, and b is the bias unit.) Now, this softmax function computes the probability that this training sample \\mathbf{x}^{(i)} belongs to class j given the weight and net input z^{(i)} . So, we compute the probability p(y = j \\mid \\mathbf{x^{(i)}; w}_j) for each class label in j = 1, \\ldots, k. . Note the normalization term in the denominator which causes these class probabilities to sum up to one. To illustrate the concept of softmax, let us walk through a concrete example. Let's assume we have a training set consisting of 4 samples from 3 different classes (0, 1, and 2) x_0 \\rightarrow \\text{class }0 x_1 \\rightarrow \\text{class }1 x_2 \\rightarrow \\text{class }2 x_3 \\rightarrow \\text{class }2 import numpy as np y = np.array([0, 1, 2, 2]) First, we want to encode the class labels into a format that we can more easily work with; we apply one-hot encoding: y_enc = (np.arange(np.max(y) + 1) == y[:, None]).astype(float) print('one-hot encoding:\\n', y_enc) one-hot encoding: [[ 1. 0. 0.] [ 0. 1. 0.] [ 0. 0. 1.] [ 0. 0. 1.]] A sample that belongs to class 0 (the first row) has a 1 in the first cell, a sample that belongs to class 2 has a 1 in the second cell of its row, and so forth. Next, let us define the feature matrix of our 4 training samples. Here, we assume that our dataset consists of 2 features; thus, we create a 4x2 dimensional matrix of our samples and features. Similarly, we create a 2x3 dimensional weight matrix (one row per feature and one column for each class). X = np.array([[0.1, 0.5], [1.1, 2.3], [-1.1, -2.3], [-1.5, -2.5]]) W = np.array([[0.1, 0.2, 0.3], [0.1, 0.2, 0.3]]) bias = np.array([0.01, 0.1, 0.1]) print('Inputs X:\\n', X) print('\\nWeights W:\\n', W) print('\\nbias:\\n', bias) Inputs X: [[ 0.1 0.5] [ 1.1 2.3] [-1.1 -2.3] [-1.5 -2.5]] Weights W: [[ 0.1 0.2 0.3] [ 0.1 0.2 0.3]] bias: [ 0.01 0.1 0.1 ] To compute the net input, we multiply the 4x2 matrix feature matrix X with the 2x3 (n_features x n_classes) weight matrix W , which yields a 4x3 output matrix (n_samples x n_classes) to which we then add the bias unit: \\mathbf{Z} = \\mathbf{X}\\mathbf{W} + \\mathbf{b}. X = np.array([[0.1, 0.5], [1.1, 2.3], [-1.1, -2.3], [-1.5, -2.5]]) W = np.array([[0.1, 0.2, 0.3], [0.1, 0.2, 0.3]]) bias = np.array([0.01, 0.1, 0.1]) print('Inputs X:\\n', X) print('\\nWeights W:\\n', W) print('\\nbias:\\n', bias) Inputs X: [[ 0.1 0.5] [ 1.1 2.3] [-1.1 -2.3] [-1.5 -2.5]] Weights W: [[ 0.1 0.2 0.3] [ 0.1 0.2 0.3]] bias: [ 0.01 0.1 0.1 ] def net_input(X, W, b): return (X.dot(W) + b) net_in = net_input(X, W, bias) print('net input:\\n', net_in) net input: [[ 0.07 0.22 0.28] [ 0.35 0.78 1.12] [-0.33 -0.58 -0.92] [-0.39 -0.7 -1.1 ]] Now, it's time to compute the softmax activation that we discussed earlier: P(y=j \\mid z^{(i)}) = \\phi_{softmax}(z^{(i)}) = \\frac{e^{z^{(i)}}}{\\sum_{j=0}^{k} e^{z_{k}^{(i)}}}. def softmax(z): return (np.exp(z.T) / np.sum(np.exp(z), axis=1)).T smax = softmax(net_in) print('softmax:\\n', smax) softmax: [[ 0.29450637 0.34216758 0.36332605] [ 0.21290077 0.32728332 0.45981591] [ 0.42860913 0.33380113 0.23758974] [ 0.44941979 0.32962558 0.22095463]] As we can see, the values for each sample (row) nicely sum up to 1 now. E.g., we can say that the first sample [ 0.29450637 0.34216758 0.36332605] has a 29.45% probability to belong to class 0. Now, in order to turn these probabilities back into class labels, we could simply take the argmax-index position of each row: [[ 0.29450637 0.34216758 0.36332605 ] -> 2 [ 0.21290077 0.32728332 0.45981591 ] -> 2 [ 0.42860913 0.33380113 0.23758974] -> 0 [ 0.44941979 0.32962558 0.22095463]] -> 0 def to_classlabel(z): return z.argmax(axis=1) print('predicted class labels: ', to_classlabel(smax)) predicted class labels: [2 2 0 0] As we can see, our predictions are terribly wrong, since the correct class labels are [0, 1, 2, 2] . Now, in order to train our logistic model (e.g., via an optimization algorithm such as gradient descent), we need to define a cost function J(\\cdot) that we want to minimize: J(\\mathbf{W}; \\mathbf{b}) = \\frac{1}{n} \\sum_{i=1}^{n} H(T_i, O_i), which is the average of all cross-entropies over our n training samples. The cross-entropy function is defined as H(T_i, O_i) = -\\sum_m T_i \\cdot log(O_i). Here the T stands for \"target\" (i.e., the true class labels) and the O stands for output -- the computed probability via softmax; not the predicted class label. def cross_entropy(output, y_target): return - np.sum(np.log(output) * (y_target), axis=1) xent = cross_entropy(smax, y_enc) print('Cross Entropy:', xent) Cross Entropy: [ 1.22245465 1.11692907 1.43720989 1.50979788] def cost(output, y_target): return np.mean(cross_entropy(output, y_target)) J_cost = cost(smax, y_enc) print('Cost: ', J_cost) Cost: 1.32159787159 In order to learn our softmax model -- determining the weight coefficients -- via gradient descent, we then need to compute the derivative \\nabla \\mathbf{w}_j \\, J(\\mathbf{W}; \\mathbf{b}). I don't want to walk through the tedious details here, but this cost derivative turns out to be simply: \\nabla \\mathbf{w}_j \\, J(\\mathbf{W}; \\mathbf{b}) = \\frac{1}{n} \\sum^{n}_{i=0} \\big[\\mathbf{x}^{(i)}\\ \\big(O_i - T_i \\big) \\big] We can then use the cost derivate to update the weights in opposite direction of the cost gradient with learning rate \\eta : \\mathbf{w}_j := \\mathbf{w}_j - \\eta \\nabla \\mathbf{w}_j \\, J(\\mathbf{W}; \\mathbf{b}) for each class j \\in \\{0, 1, ..., k\\} (note that \\mathbf{w}_j is the weight vector for the class y=j ), and we update the bias units \\mathbf{b}_j := \\mathbf{b}_j - \\eta \\bigg[ \\frac{1}{n} \\sum^{n}_{i=0} \\big(O_i - T_i \\big) \\bigg]. As a penalty against complexity, an approach to reduce the variance of our model and decrease the degree of overfitting by adding additional bias, we can further add a regularization term such as the L2 term with the regularization parameter \\lambda : L2: \\frac{\\lambda}{2} ||\\mathbf{w}||_{2}^{2} , where ||\\mathbf{w}||_{2}^{2} = \\sum^{m}_{l=0} \\sum^{k}_{j=0} w_{i, j} so that our cost function becomes J(\\mathbf{W}; \\mathbf{b}) = \\frac{1}{n} \\sum_{i=1}^{n} H(T_i, O_i) + \\frac{\\lambda}{2} ||\\mathbf{w}||_{2}^{2} and we define the \"regularized\" weight update as \\mathbf{w}_j := \\mathbf{w}_j - \\eta \\big[\\nabla \\mathbf{w}_j \\, J(\\mathbf{W}) + \\lambda \\mathbf{w}_j \\big]. (Please note that we don't regularize the bias term.)","title":"Overview"},{"location":"user_guide/classifier/SoftmaxRegression/#example-1-gradient-descent","text":"from mlxtend.data import iris_data from mlxtend.plotting import plot_decision_regions from mlxtend.classifier import SoftmaxRegression import matplotlib.pyplot as plt # Loading Data X, y = iris_data() X = X[:, [0, 3]] # sepal length and petal width # standardize X[:,0] = (X[:,0] - X[:,0].mean()) / X[:,0].std() X[:,1] = (X[:,1] - X[:,1].mean()) / X[:,1].std() lr = SoftmaxRegression(eta=0.01, epochs=500, minibatches=1, random_seed=1, print_progress=3) lr.fit(X, y) plot_decision_regions(X, y, clf=lr) plt.title('Softmax Regression - Gradient Descent') plt.show() plt.plot(range(len(lr.cost_)), lr.cost_) plt.xlabel('Iterations') plt.ylabel('Cost') plt.show() Iteration: 500/500 | Cost 0.06 | Elapsed: 0:00:00 | ETA: 0:00:00","title":"Example 1 - Gradient Descent"},{"location":"user_guide/classifier/SoftmaxRegression/#predicting-class-labels","text":"y_pred = lr.predict(X) print('Last 3 Class Labels: %s' % y_pred[-3:]) Last 3 Class Labels: [2 2 2]","title":"Predicting Class Labels"},{"location":"user_guide/classifier/SoftmaxRegression/#predicting-class-probabilities","text":"y_pred = lr.predict_proba(X) print('Last 3 Class Labels:\\n %s' % y_pred[-3:]) Last 3 Class Labels: [[ 9.18728149e-09 1.68894679e-02 9.83110523e-01] [ 2.97052325e-11 7.26356627e-04 9.99273643e-01] [ 1.57464093e-06 1.57779528e-01 8.42218897e-01]]","title":"Predicting Class Probabilities"},{"location":"user_guide/classifier/SoftmaxRegression/#example-2-stochastic-gradient-descent","text":"from mlxtend.data import iris_data from mlxtend.plotting import plot_decision_regions from mlxtend.classifier import SoftmaxRegression import matplotlib.pyplot as plt # Loading Data X, y = iris_data() X = X[:, [0, 3]] # sepal length and petal width # standardize X[:,0] = (X[:,0] - X[:,0].mean()) / X[:,0].std() X[:,1] = (X[:,1] - X[:,1].mean()) / X[:,1].std() lr = SoftmaxRegression(eta=0.01, epochs=300, minibatches=len(y), random_seed=1) lr.fit(X, y) plot_decision_regions(X, y, clf=lr) plt.title('Softmax Regression - Stochastic Gradient Descent') plt.show() plt.plot(range(len(lr.cost_)), lr.cost_) plt.xlabel('Iterations') plt.ylabel('Cost') plt.show()","title":"Example 2 - Stochastic Gradient Descent"},{"location":"user_guide/classifier/SoftmaxRegression/#api","text":"SoftmaxRegression(eta=0.01, epochs=50, l2=0.0, minibatches=1, n_classes=None, random_seed=None, print_progress=0) Softmax regression classifier. Parameters eta : float (default: 0.01) Learning rate (between 0.0 and 1.0) epochs : int (default: 50) Passes over the training dataset. Prior to each epoch, the dataset is shuffled if minibatches > 1 to prevent cycles in stochastic gradient descent. l2 : float Regularization parameter for L2 regularization. No regularization if l2=0.0. minibatches : int (default: 1) The number of minibatches for gradient-based optimization. If 1: Gradient Descent learning If len(y): Stochastic Gradient Descent (SGD) online learning If 1 < minibatches < len(y): SGD Minibatch learning n_classes : int (default: None) A positive integer to declare the number of class labels if not all class labels are present in a partial training set. Gets the number of class labels automatically if None. random_seed : int (default: None) Set random state for shuffling and initializing the weights. print_progress : int (default: 0) Prints progress in fitting to stderr. 0: No output 1: Epochs elapsed and cost 2: 1 plus time elapsed 3: 2 plus estimated time until completion Attributes w_ : 2d-array, shape={n_features, 1} Model weights after fitting. b_ : 1d-array, shape={1,} Bias unit after fitting. cost_ : list List of floats, the average cross_entropy for each epoch. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/classifier/SoftmaxRegression/","title":"API"},{"location":"user_guide/classifier/SoftmaxRegression/#methods","text":"fit(X, y, init_params=True) Learn model from training data. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values. init_params : bool (default: True) Re-initializes model parameters prior to fitting. Set False to continue training with weights from a previous model fitting. Returns self : object predict(X) Predict targets from X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns target_values : array-like, shape = [n_samples] Predicted target values. predict_proba(X) Predict class probabilities of X from the net input. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns Class probabilties : array-like, shape= [n_samples, n_classes] score(X, y) Compute the prediction accuracy Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values (true class labels). Returns acc : float The prediction accuracy as a float between 0.0 and 1.0 (perfect score). ython","title":"Methods"},{"location":"user_guide/classifier/StackingCVClassifier/","text":"StackingCVClassifier: Stacking with cross-validation An ensemble-learning meta-classifier for stacking using cross-validation to prepare the inputs for the level-2 classifier to prevent overfitting. from mlxtend.classifier import StackingCVClassifier Overview Stacking is an ensemble learning technique to combine multiple classification models via a meta-classifier. The StackingCVClassifier extends the standard stacking algorithm (implemented as StackingClassifier ) using cross-validation to prepare the input data for the level-2 classifier. In the standard stacking procedure, the first-level classifiers are fit to the same training set that is used prepare the inputs for the second-level classifier, which may lead to overfitting. The StackingCVClassifier , however, uses the concept of cross-validation: the dataset is split into k folds, and in k successive rounds, k-1 folds are used to fit the first level classifier; in each round, the first-level classifiers are then applied to the remaining 1 subset that was not used for model fitting in each iteration. The resulting predictions are then stacked and provided -- as input data -- to the second-level classifier. After the training of the StackingCVClassifier , the first-level classifiers are fit to the entire dataset as illustrated in the figure below. More formally, the Stacking Cross-Validation algorithm can be summarized as follows (source: [1]): References [1] Tang, J., S. Alelyani, and H. Liu. \" Data Classification: Algorithms and Applications. \" Data Mining and Knowledge Discovery Series, CRC Press (2015): pp. 498-500. [2] Wolpert, David H. \" Stacked generalization. \" Neural networks 5.2 (1992): 241-259. [3] Marios Michailidis (2017), StackNet, StackNet Meta Modelling Framework, https://github.com/kaz-Anova/StackNet Example 1 - Simple Stacking CV Classification from sklearn import datasets iris = datasets.load_iris() X, y = iris.data[:, 1:3], iris.target from sklearn import model_selection from sklearn.linear_model import LogisticRegression from sklearn.neighbors import KNeighborsClassifier from sklearn.naive_bayes import GaussianNB from sklearn.ensemble import RandomForestClassifier from mlxtend.classifier import StackingCVClassifier import numpy as np import warnings warnings.simplefilter('ignore') RANDOM_SEED = 42 clf1 = KNeighborsClassifier(n_neighbors=1) clf2 = RandomForestClassifier(random_state=RANDOM_SEED) clf3 = GaussianNB() lr = LogisticRegression() # Starting from v0.16.0, StackingCVRegressor supports # `random_state` to get deterministic result. sclf = StackingCVClassifier(classifiers=[clf1, clf2, clf3], meta_classifier=lr, random_state=RANDOM_SEED) print('3-fold cross validation:\\n') for clf, label in zip([clf1, clf2, clf3, sclf], ['KNN', 'Random Forest', 'Naive Bayes', 'StackingClassifier']): scores = model_selection.cross_val_score(clf, X, y, cv=3, scoring='accuracy') print(\"Accuracy: %0.2f (+/- %0.2f) [%s]\" % (scores.mean(), scores.std(), label)) 3-fold cross validation: Accuracy: 0.91 (+/- 0.01) [KNN] Accuracy: 0.95 (+/- 0.01) [Random Forest] Accuracy: 0.91 (+/- 0.02) [Naive Bayes] Accuracy: 0.93 (+/- 0.02) [StackingClassifier] import matplotlib.pyplot as plt from mlxtend.plotting import plot_decision_regions import matplotlib.gridspec as gridspec import itertools gs = gridspec.GridSpec(2, 2) fig = plt.figure(figsize=(10,8)) for clf, lab, grd in zip([clf1, clf2, clf3, sclf], ['KNN', 'Random Forest', 'Naive Bayes', 'StackingCVClassifier'], itertools.product([0, 1], repeat=2)): clf.fit(X, y) ax = plt.subplot(gs[grd[0], grd[1]]) fig = plot_decision_regions(X=X, y=y, clf=clf) plt.title(lab) plt.show() Example 2 - Using Probabilities as Meta-Features Alternatively, the class-probabilities of the first-level classifiers can be used to train the meta-classifier (2nd-level classifier) by setting use_probas=True . For example, in a 3-class setting with 2 level-1 classifiers, these classifiers may make the following \"probability\" predictions for 1 training sample: classifier 1: [0.2, 0.5, 0.3] classifier 2: [0.3, 0.4, 0.4] This results in k features, where k = [n_classes * n_classifiers], by stacking these level-1 probabilities: [0.2, 0.5, 0.3, 0.3, 0.4, 0.4] clf1 = KNeighborsClassifier(n_neighbors=1) clf2 = RandomForestClassifier(random_state=1) clf3 = GaussianNB() lr = LogisticRegression() sclf = StackingCVClassifier(classifiers=[clf1, clf2, clf3], use_probas=True, meta_classifier=lr, random_state=42) print('3-fold cross validation:\\n') for clf, label in zip([clf1, clf2, clf3, sclf], ['KNN', 'Random Forest', 'Naive Bayes', 'StackingClassifier']): scores = model_selection.cross_val_score(clf, X, y, cv=3, scoring='accuracy') print(\"Accuracy: %0.2f (+/- %0.2f) [%s]\" % (scores.mean(), scores.std(), label)) 3-fold cross validation: Accuracy: 0.91 (+/- 0.01) [KNN] Accuracy: 0.95 (+/- 0.01) [Random Forest] Accuracy: 0.91 (+/- 0.02) [Naive Bayes] Accuracy: 0.95 (+/- 0.02) [StackingClassifier] Example 3 - Stacked CV Classification and GridSearch The stack allows tuning hyper parameters of the base and meta models! A full list of tunable parameters can be obtained via estimator.get_params().keys() . from sklearn.linear_model import LogisticRegression from sklearn.neighbors import KNeighborsClassifier from sklearn.naive_bayes import GaussianNB from sklearn.ensemble import RandomForestClassifier from sklearn.model_selection import GridSearchCV from mlxtend.classifier import StackingCVClassifier # Initializing models clf1 = KNeighborsClassifier(n_neighbors=1) clf2 = RandomForestClassifier(random_state=RANDOM_SEED) clf3 = GaussianNB() lr = LogisticRegression() sclf = StackingCVClassifier(classifiers=[clf1, clf2, clf3], meta_classifier=lr, random_state=42) params = {'kneighborsclassifier__n_neighbors': [1, 5], 'randomforestclassifier__n_estimators': [10, 50], 'meta_classifier__C': [0.1, 10.0]} grid = GridSearchCV(estimator=sclf, param_grid=params, cv=5, refit=True) grid.fit(X, y) cv_keys = ('mean_test_score', 'std_test_score', 'params') for r, _ in enumerate(grid.cv_results_['mean_test_score']): print(\"%0.3f +/- %0.2f %r\" % (grid.cv_results_[cv_keys[0]][r], grid.cv_results_[cv_keys[1]][r] / 2.0, grid.cv_results_[cv_keys[2]][r])) print('Best parameters: %s' % grid.best_params_) print('Accuracy: %.2f' % grid.best_score_) 0.947 +/- 0.03 {'kneighborsclassifier__n_neighbors': 1, 'meta_classifier__C': 0.1, 'randomforestclassifier__n_estimators': 10} 0.933 +/- 0.02 {'kneighborsclassifier__n_neighbors': 1, 'meta_classifier__C': 0.1, 'randomforestclassifier__n_estimators': 50} 0.940 +/- 0.02 {'kneighborsclassifier__n_neighbors': 1, 'meta_classifier__C': 10.0, 'randomforestclassifier__n_estimators': 10} 0.940 +/- 0.02 {'kneighborsclassifier__n_neighbors': 1, 'meta_classifier__C': 10.0, 'randomforestclassifier__n_estimators': 50} 0.953 +/- 0.02 {'kneighborsclassifier__n_neighbors': 5, 'meta_classifier__C': 0.1, 'randomforestclassifier__n_estimators': 10} 0.953 +/- 0.02 {'kneighborsclassifier__n_neighbors': 5, 'meta_classifier__C': 0.1, 'randomforestclassifier__n_estimators': 50} 0.953 +/- 0.02 {'kneighborsclassifier__n_neighbors': 5, 'meta_classifier__C': 10.0, 'randomforestclassifier__n_estimators': 10} 0.953 +/- 0.02 {'kneighborsclassifier__n_neighbors': 5, 'meta_classifier__C': 10.0, 'randomforestclassifier__n_estimators': 50} Best parameters: {'kneighborsclassifier__n_neighbors': 5, 'meta_classifier__C': 0.1, 'randomforestclassifier__n_estimators': 10} Accuracy: 0.95 In case we are planning to use a regression algorithm multiple times, all we need to do is to add an additional number suffix in the parameter grid as shown below: from sklearn.model_selection import GridSearchCV # Initializing models clf1 = KNeighborsClassifier(n_neighbors=1) clf2 = RandomForestClassifier(random_state=RANDOM_SEED) clf3 = GaussianNB() lr = LogisticRegression() sclf = StackingCVClassifier(classifiers=[clf1, clf1, clf2, clf3], meta_classifier=lr, random_state=RANDOM_SEED) params = {'kneighborsclassifier-1__n_neighbors': [1, 5], 'kneighborsclassifier-2__n_neighbors': [1, 5], 'randomforestclassifier__n_estimators': [10, 50], 'meta_classifier__C': [0.1, 10.0]} grid = GridSearchCV(estimator=sclf, param_grid=params, cv=5, refit=True) grid.fit(X, y) cv_keys = ('mean_test_score', 'std_test_score', 'params') for r, _ in enumerate(grid.cv_results_['mean_test_score']): print(\"%0.3f +/- %0.2f %r\" % (grid.cv_results_[cv_keys[0]][r], grid.cv_results_[cv_keys[1]][r] / 2.0, grid.cv_results_[cv_keys[2]][r])) print('Best parameters: %s' % grid.best_params_) print('Accuracy: %.2f' % grid.best_score_) 0.940 +/- 0.02 {'kneighborsclassifier-1__n_neighbors': 1, 'kneighborsclassifier-2__n_neighbors': 1, 'meta_classifier__C': 0.1, 'randomforestclassifier__n_estimators': 10} 0.940 +/- 0.02 {'kneighborsclassifier-1__n_neighbors': 1, 'kneighborsclassifier-2__n_neighbors': 1, 'meta_classifier__C': 0.1, 'randomforestclassifier__n_estimators': 50} 0.940 +/- 0.02 {'kneighborsclassifier-1__n_neighbors': 1, 'kneighborsclassifier-2__n_neighbors': 1, 'meta_classifier__C': 10.0, 'randomforestclassifier__n_estimators': 10} 0.940 +/- 0.02 {'kneighborsclassifier-1__n_neighbors': 1, 'kneighborsclassifier-2__n_neighbors': 1, 'meta_classifier__C': 10.0, 'randomforestclassifier__n_estimators': 50} 0.960 +/- 0.02 {'kneighborsclassifier-1__n_neighbors': 1, 'kneighborsclassifier-2__n_neighbors': 5, 'meta_classifier__C': 0.1, 'randomforestclassifier__n_estimators': 10} 0.953 +/- 0.02 {'kneighborsclassifier-1__n_neighbors': 1, 'kneighborsclassifier-2__n_neighbors': 5, 'meta_classifier__C': 0.1, 'randomforestclassifier__n_estimators': 50} 0.953 +/- 0.02 {'kneighborsclassifier-1__n_neighbors': 1, 'kneighborsclassifier-2__n_neighbors': 5, 'meta_classifier__C': 10.0, 'randomforestclassifier__n_estimators': 10} 0.953 +/- 0.02 {'kneighborsclassifier-1__n_neighbors': 1, 'kneighborsclassifier-2__n_neighbors': 5, 'meta_classifier__C': 10.0, 'randomforestclassifier__n_estimators': 50} 0.960 +/- 0.02 {'kneighborsclassifier-1__n_neighbors': 5, 'kneighborsclassifier-2__n_neighbors': 1, 'meta_classifier__C': 0.1, 'randomforestclassifier__n_estimators': 10} 0.953 +/- 0.02 {'kneighborsclassifier-1__n_neighbors': 5, 'kneighborsclassifier-2__n_neighbors': 1, 'meta_classifier__C': 0.1, 'randomforestclassifier__n_estimators': 50} 0.953 +/- 0.02 {'kneighborsclassifier-1__n_neighbors': 5, 'kneighborsclassifier-2__n_neighbors': 1, 'meta_classifier__C': 10.0, 'randomforestclassifier__n_estimators': 10} 0.953 +/- 0.02 {'kneighborsclassifier-1__n_neighbors': 5, 'kneighborsclassifier-2__n_neighbors': 1, 'meta_classifier__C': 10.0, 'randomforestclassifier__n_estimators': 50} 0.953 +/- 0.02 {'kneighborsclassifier-1__n_neighbors': 5, 'kneighborsclassifier-2__n_neighbors': 5, 'meta_classifier__C': 0.1, 'randomforestclassifier__n_estimators': 10} 0.953 +/- 0.02 {'kneighborsclassifier-1__n_neighbors': 5, 'kneighborsclassifier-2__n_neighbors': 5, 'meta_classifier__C': 0.1, 'randomforestclassifier__n_estimators': 50} 0.953 +/- 0.02 {'kneighborsclassifier-1__n_neighbors': 5, 'kneighborsclassifier-2__n_neighbors': 5, 'meta_classifier__C': 10.0, 'randomforestclassifier__n_estimators': 10} 0.953 +/- 0.02 {'kneighborsclassifier-1__n_neighbors': 5, 'kneighborsclassifier-2__n_neighbors': 5, 'meta_classifier__C': 10.0, 'randomforestclassifier__n_estimators': 50} Best parameters: {'kneighborsclassifier-1__n_neighbors': 1, 'kneighborsclassifier-2__n_neighbors': 5, 'meta_classifier__C': 0.1, 'randomforestclassifier__n_estimators': 10} Accuracy: 0.96 Note The StackingClassifier also enables grid search over the classifiers argument. When there are level-mixed hyperparameters, GridSearchCV will try to replace hyperparameters in a top-down order, i.e., classifers -> single base classifier -> classifier hyperparameter. For instance, given a hyperparameter grid such as params = {'randomforestclassifier__n_estimators': [1, 100], 'classifiers': [(clf1, clf1, clf1), (clf2, clf3)]} it will first use the instance settings of either (clf1, clf1, clf1) or (clf2, clf3). Then it will replace the 'n_estimators' settings for a matching classifier based on 'randomforestclassifier__n_estimators': [1, 100] . Example 4 - Stacking of Classifiers that Operate on Different Feature Subsets The different level-1 classifiers can be fit to different subsets of features in the training dataset. The following example illustrates how this can be done on a technical level using scikit-learn pipelines and the ColumnSelector : from sklearn.datasets import load_iris from mlxtend.classifier import StackingCVClassifier from mlxtend.feature_selection import ColumnSelector from sklearn.pipeline import make_pipeline from sklearn.linear_model import LogisticRegression iris = load_iris() X = iris.data y = iris.target pipe1 = make_pipeline(ColumnSelector(cols=(0, 2)), LogisticRegression()) pipe2 = make_pipeline(ColumnSelector(cols=(1, 2, 3)), LogisticRegression()) sclf = StackingCVClassifier(classifiers=[pipe1, pipe2], meta_classifier=LogisticRegression(), random_state=42) sclf.fit(X, y) StackingCVClassifier(classifiers=[Pipeline(memory=None, steps=[('columnselector', ColumnSelector(cols=(0, 2), drop_axis=False)), ('logisticregression', LogisticRegression(C=1.0, class_weight=None, dual=False, fit_intercept=True, intercept_scaling=1, l1_ratio=None, max_iter=100, multi_class='auto', n_jobs=None, penalty='l2', random_state=None, solver='lbfgs', tol=0.0... fit_intercept=True, intercept_scaling=1, l1_ratio=None, max_iter=100, multi_class='auto', n_jobs=None, penalty='l2', random_state=None, solver='lbfgs', tol=0.0001, verbose=0, warm_start=False), n_jobs=None, pre_dispatch='2*n_jobs', random_state=42, shuffle=True, store_train_meta_features=False, stratify=True, use_clones=True, use_features_in_secondary=False, use_probas=False, verbose=0) Example 5 -- ROC Curve with decision_function Like other scikit-learn classifiers, the StackingCVClassifier has an decision_function method that can be used for plotting ROC curves. Note that the decision_function expects and requires the meta-classifier to implement a decision_function . from sklearn import model_selection from sklearn.linear_model import LogisticRegression from sklearn.neighbors import KNeighborsClassifier from sklearn.svm import SVC from sklearn.ensemble import RandomForestClassifier from mlxtend.classifier import StackingCVClassifier from sklearn.metrics import roc_curve, auc import numpy as np from sklearn.model_selection import train_test_split from sklearn import datasets from sklearn.preprocessing import label_binarize from sklearn.multiclass import OneVsRestClassifier iris = datasets.load_iris() X, y = iris.data[:, [0, 1]], iris.target # Binarize the output y = label_binarize(y, classes=[0, 1, 2]) n_classes = y.shape[1] RANDOM_SEED = 42 X_train, X_test, y_train, y_test = train_test_split( X, y, test_size=0.33, random_state=RANDOM_SEED) clf1 = LogisticRegression() clf2 = RandomForestClassifier(random_state=RANDOM_SEED) clf3 = SVC(random_state=RANDOM_SEED) lr = LogisticRegression() sclf = StackingCVClassifier(classifiers=[clf1, clf2, clf3], meta_classifier=lr) # Learn to predict each class against the other classifier = OneVsRestClassifier(sclf) Using predict_proba() y_score = classifier.fit(X_train, y_train).predict_proba(X_test) # Compute ROC curve and ROC area for each class fpr = dict() tpr = dict() roc_auc = dict() for i in range(n_classes): fpr[i], tpr[i], _ = roc_curve(y_test[:, i], y_score[:, i]) roc_auc[i] = auc(fpr[i], tpr[i]) # Compute micro-average ROC curve and ROC area fpr[\"micro\"], tpr[\"micro\"], _ = roc_curve(y_test.ravel(), y_score.ravel()) roc_auc[\"micro\"] = auc(fpr[\"micro\"], tpr[\"micro\"]) plt.figure() lw = 2 plt.plot(fpr[2], tpr[2], color='darkorange', lw=lw, label='ROC curve (area = %0.2f)' % roc_auc[2]) plt.plot([0, 1], [0, 1], color='navy', lw=lw, linestyle='--') plt.xlim([0.0, 1.0]) plt.ylim([0.0, 1.05]) plt.xlabel('False Positive Rate') plt.ylabel('True Positive Rate') plt.title('Receiver operating characteristic example') plt.legend(loc=\"lower right\") plt.show() Using decision_function() y_score = classifier.fit(X_train, y_train).decision_function(X_test) # Compute ROC curve and ROC area for each class fpr = dict() tpr = dict() roc_auc = dict() for i in range(n_classes): fpr[i], tpr[i], _ = roc_curve(y_test[:, i], y_score[:, i]) roc_auc[i] = auc(fpr[i], tpr[i]) # Compute micro-average ROC curve and ROC area fpr[\"micro\"], tpr[\"micro\"], _ = roc_curve(y_test.ravel(), y_score.ravel()) roc_auc[\"micro\"] = auc(fpr[\"micro\"], tpr[\"micro\"]) plt.figure() lw = 2 plt.plot(fpr[2], tpr[2], color='darkorange', lw=lw, label='ROC curve (area = %0.2f)' % roc_auc[2]) plt.plot([0, 1], [0, 1], color='navy', lw=lw, linestyle='--') plt.xlim([0.0, 1.0]) plt.ylim([0.0, 1.05]) plt.xlabel('False Positive Rate') plt.ylabel('True Positive Rate') plt.title('Receiver operating characteristic example') plt.legend(loc=\"lower right\") plt.show() API StackingCVClassifier(classifiers, meta_classifier, use_probas=False, drop_proba_col=None, cv=2, shuffle=True, random_state=None, stratify=True, verbose=0, use_features_in_secondary=False, store_train_meta_features=False, use_clones=True, n_jobs=None, pre_dispatch='2 n_jobs')* A 'Stacking Cross-Validation' classifier for scikit-learn estimators. New in mlxtend v0.4.3 Parameters classifiers : array-like, shape = [n_classifiers] A list of classifiers. Invoking the fit method on the StackingCVClassifer will fit clones of these original classifiers that will be stored in the class attribute self.clfs_ . meta_classifier : object The meta-classifier to be fitted on the ensemble of classifiers use_probas : bool (default: False) If True, trains meta-classifier based on predicted probabilities instead of class labels. drop_proba_col : string (default: None) Drops extra \"probability\" column in the feature set, because it is redundant: p(y_c) = 1 - p(y_1) + p(y_2) + ... + p(y_{c-1}). This can be useful for meta-classifiers that are sensitive to perfectly collinear features. If last , drops last probability column. If first , drops first probability column. Only relevant if use_probas=True . cv : int, cross-validation generator or an iterable, optional (default: 2) Determines the cross-validation splitting strategy. Possible inputs for cv are: - None, to use the default 2-fold cross validation, - integer, to specify the number of folds in a (Stratified)KFold , - An object to be used as a cross-validation generator. - An iterable yielding train, test splits. For integer/None inputs, it will use either a KFold or StratifiedKFold cross validation depending the value of stratify argument. shuffle : bool (default: True) If True, and the cv argument is integer, the training data will be shuffled at fitting stage prior to cross-validation. If the cv argument is a specific cross validation technique, this argument is omitted. random_state : int, RandomState instance or None, optional (default: None) Constrols the randomness of the cv splitter. Used when cv is integer and shuffle=True . New in v0.16.0. stratify : bool (default: True) If True, and the cv argument is integer it will follow a stratified K-Fold cross validation technique. If the cv argument is a specific cross validation technique, this argument is omitted. verbose : int, optional (default=0) Controls the verbosity of the building process. - verbose=0 (default): Prints nothing - verbose=1 : Prints the number & name of the regressor being fitted and which fold is currently being used for fitting - verbose=2 : Prints info about the parameters of the regressor being fitted - verbose>2 : Changes verbose param of the underlying regressor to self.verbose - 2 use_features_in_secondary : bool (default: False) If True, the meta-classifier will be trained both on the predictions of the original classifiers and the original dataset. If False, the meta-classifier will be trained only on the predictions of the original classifiers. store_train_meta_features : bool (default: False) If True, the meta-features computed from the training data used for fitting the meta-classifier stored in the self.train_meta_features_ array, which can be accessed after calling fit . use_clones : bool (default: True) Clones the classifiers for stacking classification if True (default) or else uses the original ones, which will be refitted on the dataset upon calling the fit method. Hence, if use_clones=True, the original input classifiers will remain unmodified upon using the StackingCVClassifier's fit method. Setting use_clones=False is recommended if you are working with estimators that are supporting the scikit-learn fit/predict API interface but are not compatible to scikit-learn's clone function. n_jobs : int or None, optional (default=None) The number of CPUs to use to do the computation. None means 1 unless in a :obj: joblib.parallel_backend context. -1 means using all processors. See :term: Glossary <n_jobs> for more details. New in v0.16.0. pre_dispatch : int, or string, optional Controls the number of jobs that get dispatched during parallel execution. Reducing this number can be useful to avoid an explosion of memory consumption when more jobs get dispatched than CPUs can process. This parameter can be: - None, in which case all the jobs are immediately created and spawned. Use this for lightweight and fast-running jobs, to avoid delays due to on-demand spawning of the jobs - An int, giving the exact number of total jobs that are spawned - A string, giving an expression as a function of n_jobs, as in '2*n_jobs' New in v0.16.0. Attributes clfs_ : list, shape=[n_classifiers] Fitted classifiers (clones of the original classifiers) meta_clf_ : estimator Fitted meta-classifier (clone of the original meta-estimator) train_meta_features : numpy array, shape = [n_samples, n_classifiers] meta-features for training data, where n_samples is the number of samples in training data and n_classifiers is the number of classfiers. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/classifier/StackingCVClassifier/ Methods decision_function(X) Predict class confidence scores for X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns scores : shape=(n_samples,) if n_classes == 2 else (n_samples, n_classes). Confidence scores per (sample, class) combination. In the binary case, confidence score for self.classes_[1] where >0 means this class would be predicted. fit(X, y, groups=None, sample_weight=None) Fit ensemble classifers and the meta-classifier. Parameters X : numpy array, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : numpy array, shape = [n_samples] Target values. groups : numpy array/None, shape = [n_samples] The group that each sample belongs to. This is used by specific folding strategies such as GroupKFold() sample_weight : array-like, shape = [n_samples], optional Sample weights passed as sample_weights to each regressor in the regressors list as well as the meta_regressor. Raises error if some regressor does not support sample_weight in the fit() method. Returns self : object fit_transform(X, y=None, fit_params) Fit to data, then transform it. Fits transformer to X and y with optional parameters fit_params and returns a transformed version of X. Parameters X : numpy array of shape [n_samples, n_features] Training set. y : numpy array of shape [n_samples] Target values. **fit_params : dict Additional fit parameters. Returns X_new : numpy array of shape [n_samples, n_features_new] Transformed array. get_params(deep=True) Return estimator parameter names for GridSearch support. predict(X) Predict target values for X. Parameters X : numpy array, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns labels : array-like, shape = [n_samples] Predicted class labels. predict_meta_features(X) Get meta-features of test-data. Parameters X : numpy array, shape = [n_samples, n_features] Test vectors, where n_samples is the number of samples and n_features is the number of features. Returns meta-features : numpy array, shape = [n_samples, n_classifiers] Returns the meta-features for test data. predict_proba(X) Predict class probabilities for X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns proba : array-like, shape = [n_samples, n_classes] or a list of n_outputs of such arrays if n_outputs > 1. Probability for each class per sample. score(X, y, sample_weight=None) Return the mean accuracy on the given test data and labels. In multi-label classification, this is the subset accuracy which is a harsh metric since you require for each sample that each label set be correctly predicted. Parameters X : array-like of shape (n_samples, n_features) Test samples. y : array-like of shape (n_samples,) or (n_samples, n_outputs) True labels for X. sample_weight : array-like of shape (n_samples,), default=None Sample weights. Returns score : float Mean accuracy of self.predict(X) wrt. y. set_params( params) Set the parameters of this estimator. Valid parameter keys can be listed with get_params() . Returns self Properties named_classifiers None","title":"StackingCVClassifier: Stacking with cross-validation"},{"location":"user_guide/classifier/StackingCVClassifier/#stackingcvclassifier-stacking-with-cross-validation","text":"An ensemble-learning meta-classifier for stacking using cross-validation to prepare the inputs for the level-2 classifier to prevent overfitting. from mlxtend.classifier import StackingCVClassifier","title":"StackingCVClassifier: Stacking with cross-validation"},{"location":"user_guide/classifier/StackingCVClassifier/#overview","text":"Stacking is an ensemble learning technique to combine multiple classification models via a meta-classifier. The StackingCVClassifier extends the standard stacking algorithm (implemented as StackingClassifier ) using cross-validation to prepare the input data for the level-2 classifier. In the standard stacking procedure, the first-level classifiers are fit to the same training set that is used prepare the inputs for the second-level classifier, which may lead to overfitting. The StackingCVClassifier , however, uses the concept of cross-validation: the dataset is split into k folds, and in k successive rounds, k-1 folds are used to fit the first level classifier; in each round, the first-level classifiers are then applied to the remaining 1 subset that was not used for model fitting in each iteration. The resulting predictions are then stacked and provided -- as input data -- to the second-level classifier. After the training of the StackingCVClassifier , the first-level classifiers are fit to the entire dataset as illustrated in the figure below. More formally, the Stacking Cross-Validation algorithm can be summarized as follows (source: [1]):","title":"Overview"},{"location":"user_guide/classifier/StackingCVClassifier/#references","text":"[1] Tang, J., S. Alelyani, and H. Liu. \" Data Classification: Algorithms and Applications. \" Data Mining and Knowledge Discovery Series, CRC Press (2015): pp. 498-500. [2] Wolpert, David H. \" Stacked generalization. \" Neural networks 5.2 (1992): 241-259. [3] Marios Michailidis (2017), StackNet, StackNet Meta Modelling Framework, https://github.com/kaz-Anova/StackNet","title":"References"},{"location":"user_guide/classifier/StackingCVClassifier/#example-1-simple-stacking-cv-classification","text":"from sklearn import datasets iris = datasets.load_iris() X, y = iris.data[:, 1:3], iris.target from sklearn import model_selection from sklearn.linear_model import LogisticRegression from sklearn.neighbors import KNeighborsClassifier from sklearn.naive_bayes import GaussianNB from sklearn.ensemble import RandomForestClassifier from mlxtend.classifier import StackingCVClassifier import numpy as np import warnings warnings.simplefilter('ignore') RANDOM_SEED = 42 clf1 = KNeighborsClassifier(n_neighbors=1) clf2 = RandomForestClassifier(random_state=RANDOM_SEED) clf3 = GaussianNB() lr = LogisticRegression() # Starting from v0.16.0, StackingCVRegressor supports # `random_state` to get deterministic result. sclf = StackingCVClassifier(classifiers=[clf1, clf2, clf3], meta_classifier=lr, random_state=RANDOM_SEED) print('3-fold cross validation:\\n') for clf, label in zip([clf1, clf2, clf3, sclf], ['KNN', 'Random Forest', 'Naive Bayes', 'StackingClassifier']): scores = model_selection.cross_val_score(clf, X, y, cv=3, scoring='accuracy') print(\"Accuracy: %0.2f (+/- %0.2f) [%s]\" % (scores.mean(), scores.std(), label)) 3-fold cross validation: Accuracy: 0.91 (+/- 0.01) [KNN] Accuracy: 0.95 (+/- 0.01) [Random Forest] Accuracy: 0.91 (+/- 0.02) [Naive Bayes] Accuracy: 0.93 (+/- 0.02) [StackingClassifier] import matplotlib.pyplot as plt from mlxtend.plotting import plot_decision_regions import matplotlib.gridspec as gridspec import itertools gs = gridspec.GridSpec(2, 2) fig = plt.figure(figsize=(10,8)) for clf, lab, grd in zip([clf1, clf2, clf3, sclf], ['KNN', 'Random Forest', 'Naive Bayes', 'StackingCVClassifier'], itertools.product([0, 1], repeat=2)): clf.fit(X, y) ax = plt.subplot(gs[grd[0], grd[1]]) fig = plot_decision_regions(X=X, y=y, clf=clf) plt.title(lab) plt.show()","title":"Example 1 - Simple Stacking CV Classification"},{"location":"user_guide/classifier/StackingCVClassifier/#example-2-using-probabilities-as-meta-features","text":"Alternatively, the class-probabilities of the first-level classifiers can be used to train the meta-classifier (2nd-level classifier) by setting use_probas=True . For example, in a 3-class setting with 2 level-1 classifiers, these classifiers may make the following \"probability\" predictions for 1 training sample: classifier 1: [0.2, 0.5, 0.3] classifier 2: [0.3, 0.4, 0.4] This results in k features, where k = [n_classes * n_classifiers], by stacking these level-1 probabilities: [0.2, 0.5, 0.3, 0.3, 0.4, 0.4] clf1 = KNeighborsClassifier(n_neighbors=1) clf2 = RandomForestClassifier(random_state=1) clf3 = GaussianNB() lr = LogisticRegression() sclf = StackingCVClassifier(classifiers=[clf1, clf2, clf3], use_probas=True, meta_classifier=lr, random_state=42) print('3-fold cross validation:\\n') for clf, label in zip([clf1, clf2, clf3, sclf], ['KNN', 'Random Forest', 'Naive Bayes', 'StackingClassifier']): scores = model_selection.cross_val_score(clf, X, y, cv=3, scoring='accuracy') print(\"Accuracy: %0.2f (+/- %0.2f) [%s]\" % (scores.mean(), scores.std(), label)) 3-fold cross validation: Accuracy: 0.91 (+/- 0.01) [KNN] Accuracy: 0.95 (+/- 0.01) [Random Forest] Accuracy: 0.91 (+/- 0.02) [Naive Bayes] Accuracy: 0.95 (+/- 0.02) [StackingClassifier]","title":"Example 2 - Using Probabilities as Meta-Features"},{"location":"user_guide/classifier/StackingCVClassifier/#example-3-stacked-cv-classification-and-gridsearch","text":"The stack allows tuning hyper parameters of the base and meta models! A full list of tunable parameters can be obtained via estimator.get_params().keys() . from sklearn.linear_model import LogisticRegression from sklearn.neighbors import KNeighborsClassifier from sklearn.naive_bayes import GaussianNB from sklearn.ensemble import RandomForestClassifier from sklearn.model_selection import GridSearchCV from mlxtend.classifier import StackingCVClassifier # Initializing models clf1 = KNeighborsClassifier(n_neighbors=1) clf2 = RandomForestClassifier(random_state=RANDOM_SEED) clf3 = GaussianNB() lr = LogisticRegression() sclf = StackingCVClassifier(classifiers=[clf1, clf2, clf3], meta_classifier=lr, random_state=42) params = {'kneighborsclassifier__n_neighbors': [1, 5], 'randomforestclassifier__n_estimators': [10, 50], 'meta_classifier__C': [0.1, 10.0]} grid = GridSearchCV(estimator=sclf, param_grid=params, cv=5, refit=True) grid.fit(X, y) cv_keys = ('mean_test_score', 'std_test_score', 'params') for r, _ in enumerate(grid.cv_results_['mean_test_score']): print(\"%0.3f +/- %0.2f %r\" % (grid.cv_results_[cv_keys[0]][r], grid.cv_results_[cv_keys[1]][r] / 2.0, grid.cv_results_[cv_keys[2]][r])) print('Best parameters: %s' % grid.best_params_) print('Accuracy: %.2f' % grid.best_score_) 0.947 +/- 0.03 {'kneighborsclassifier__n_neighbors': 1, 'meta_classifier__C': 0.1, 'randomforestclassifier__n_estimators': 10} 0.933 +/- 0.02 {'kneighborsclassifier__n_neighbors': 1, 'meta_classifier__C': 0.1, 'randomforestclassifier__n_estimators': 50} 0.940 +/- 0.02 {'kneighborsclassifier__n_neighbors': 1, 'meta_classifier__C': 10.0, 'randomforestclassifier__n_estimators': 10} 0.940 +/- 0.02 {'kneighborsclassifier__n_neighbors': 1, 'meta_classifier__C': 10.0, 'randomforestclassifier__n_estimators': 50} 0.953 +/- 0.02 {'kneighborsclassifier__n_neighbors': 5, 'meta_classifier__C': 0.1, 'randomforestclassifier__n_estimators': 10} 0.953 +/- 0.02 {'kneighborsclassifier__n_neighbors': 5, 'meta_classifier__C': 0.1, 'randomforestclassifier__n_estimators': 50} 0.953 +/- 0.02 {'kneighborsclassifier__n_neighbors': 5, 'meta_classifier__C': 10.0, 'randomforestclassifier__n_estimators': 10} 0.953 +/- 0.02 {'kneighborsclassifier__n_neighbors': 5, 'meta_classifier__C': 10.0, 'randomforestclassifier__n_estimators': 50} Best parameters: {'kneighborsclassifier__n_neighbors': 5, 'meta_classifier__C': 0.1, 'randomforestclassifier__n_estimators': 10} Accuracy: 0.95 In case we are planning to use a regression algorithm multiple times, all we need to do is to add an additional number suffix in the parameter grid as shown below: from sklearn.model_selection import GridSearchCV # Initializing models clf1 = KNeighborsClassifier(n_neighbors=1) clf2 = RandomForestClassifier(random_state=RANDOM_SEED) clf3 = GaussianNB() lr = LogisticRegression() sclf = StackingCVClassifier(classifiers=[clf1, clf1, clf2, clf3], meta_classifier=lr, random_state=RANDOM_SEED) params = {'kneighborsclassifier-1__n_neighbors': [1, 5], 'kneighborsclassifier-2__n_neighbors': [1, 5], 'randomforestclassifier__n_estimators': [10, 50], 'meta_classifier__C': [0.1, 10.0]} grid = GridSearchCV(estimator=sclf, param_grid=params, cv=5, refit=True) grid.fit(X, y) cv_keys = ('mean_test_score', 'std_test_score', 'params') for r, _ in enumerate(grid.cv_results_['mean_test_score']): print(\"%0.3f +/- %0.2f %r\" % (grid.cv_results_[cv_keys[0]][r], grid.cv_results_[cv_keys[1]][r] / 2.0, grid.cv_results_[cv_keys[2]][r])) print('Best parameters: %s' % grid.best_params_) print('Accuracy: %.2f' % grid.best_score_) 0.940 +/- 0.02 {'kneighborsclassifier-1__n_neighbors': 1, 'kneighborsclassifier-2__n_neighbors': 1, 'meta_classifier__C': 0.1, 'randomforestclassifier__n_estimators': 10} 0.940 +/- 0.02 {'kneighborsclassifier-1__n_neighbors': 1, 'kneighborsclassifier-2__n_neighbors': 1, 'meta_classifier__C': 0.1, 'randomforestclassifier__n_estimators': 50} 0.940 +/- 0.02 {'kneighborsclassifier-1__n_neighbors': 1, 'kneighborsclassifier-2__n_neighbors': 1, 'meta_classifier__C': 10.0, 'randomforestclassifier__n_estimators': 10} 0.940 +/- 0.02 {'kneighborsclassifier-1__n_neighbors': 1, 'kneighborsclassifier-2__n_neighbors': 1, 'meta_classifier__C': 10.0, 'randomforestclassifier__n_estimators': 50} 0.960 +/- 0.02 {'kneighborsclassifier-1__n_neighbors': 1, 'kneighborsclassifier-2__n_neighbors': 5, 'meta_classifier__C': 0.1, 'randomforestclassifier__n_estimators': 10} 0.953 +/- 0.02 {'kneighborsclassifier-1__n_neighbors': 1, 'kneighborsclassifier-2__n_neighbors': 5, 'meta_classifier__C': 0.1, 'randomforestclassifier__n_estimators': 50} 0.953 +/- 0.02 {'kneighborsclassifier-1__n_neighbors': 1, 'kneighborsclassifier-2__n_neighbors': 5, 'meta_classifier__C': 10.0, 'randomforestclassifier__n_estimators': 10} 0.953 +/- 0.02 {'kneighborsclassifier-1__n_neighbors': 1, 'kneighborsclassifier-2__n_neighbors': 5, 'meta_classifier__C': 10.0, 'randomforestclassifier__n_estimators': 50} 0.960 +/- 0.02 {'kneighborsclassifier-1__n_neighbors': 5, 'kneighborsclassifier-2__n_neighbors': 1, 'meta_classifier__C': 0.1, 'randomforestclassifier__n_estimators': 10} 0.953 +/- 0.02 {'kneighborsclassifier-1__n_neighbors': 5, 'kneighborsclassifier-2__n_neighbors': 1, 'meta_classifier__C': 0.1, 'randomforestclassifier__n_estimators': 50} 0.953 +/- 0.02 {'kneighborsclassifier-1__n_neighbors': 5, 'kneighborsclassifier-2__n_neighbors': 1, 'meta_classifier__C': 10.0, 'randomforestclassifier__n_estimators': 10} 0.953 +/- 0.02 {'kneighborsclassifier-1__n_neighbors': 5, 'kneighborsclassifier-2__n_neighbors': 1, 'meta_classifier__C': 10.0, 'randomforestclassifier__n_estimators': 50} 0.953 +/- 0.02 {'kneighborsclassifier-1__n_neighbors': 5, 'kneighborsclassifier-2__n_neighbors': 5, 'meta_classifier__C': 0.1, 'randomforestclassifier__n_estimators': 10} 0.953 +/- 0.02 {'kneighborsclassifier-1__n_neighbors': 5, 'kneighborsclassifier-2__n_neighbors': 5, 'meta_classifier__C': 0.1, 'randomforestclassifier__n_estimators': 50} 0.953 +/- 0.02 {'kneighborsclassifier-1__n_neighbors': 5, 'kneighborsclassifier-2__n_neighbors': 5, 'meta_classifier__C': 10.0, 'randomforestclassifier__n_estimators': 10} 0.953 +/- 0.02 {'kneighborsclassifier-1__n_neighbors': 5, 'kneighborsclassifier-2__n_neighbors': 5, 'meta_classifier__C': 10.0, 'randomforestclassifier__n_estimators': 50} Best parameters: {'kneighborsclassifier-1__n_neighbors': 1, 'kneighborsclassifier-2__n_neighbors': 5, 'meta_classifier__C': 0.1, 'randomforestclassifier__n_estimators': 10} Accuracy: 0.96 Note The StackingClassifier also enables grid search over the classifiers argument. When there are level-mixed hyperparameters, GridSearchCV will try to replace hyperparameters in a top-down order, i.e., classifers -> single base classifier -> classifier hyperparameter. For instance, given a hyperparameter grid such as params = {'randomforestclassifier__n_estimators': [1, 100], 'classifiers': [(clf1, clf1, clf1), (clf2, clf3)]} it will first use the instance settings of either (clf1, clf1, clf1) or (clf2, clf3). Then it will replace the 'n_estimators' settings for a matching classifier based on 'randomforestclassifier__n_estimators': [1, 100] .","title":"Example 3 - Stacked CV Classification and GridSearch"},{"location":"user_guide/classifier/StackingCVClassifier/#example-4-stacking-of-classifiers-that-operate-on-different-feature-subsets","text":"The different level-1 classifiers can be fit to different subsets of features in the training dataset. The following example illustrates how this can be done on a technical level using scikit-learn pipelines and the ColumnSelector : from sklearn.datasets import load_iris from mlxtend.classifier import StackingCVClassifier from mlxtend.feature_selection import ColumnSelector from sklearn.pipeline import make_pipeline from sklearn.linear_model import LogisticRegression iris = load_iris() X = iris.data y = iris.target pipe1 = make_pipeline(ColumnSelector(cols=(0, 2)), LogisticRegression()) pipe2 = make_pipeline(ColumnSelector(cols=(1, 2, 3)), LogisticRegression()) sclf = StackingCVClassifier(classifiers=[pipe1, pipe2], meta_classifier=LogisticRegression(), random_state=42) sclf.fit(X, y) StackingCVClassifier(classifiers=[Pipeline(memory=None, steps=[('columnselector', ColumnSelector(cols=(0, 2), drop_axis=False)), ('logisticregression', LogisticRegression(C=1.0, class_weight=None, dual=False, fit_intercept=True, intercept_scaling=1, l1_ratio=None, max_iter=100, multi_class='auto', n_jobs=None, penalty='l2', random_state=None, solver='lbfgs', tol=0.0... fit_intercept=True, intercept_scaling=1, l1_ratio=None, max_iter=100, multi_class='auto', n_jobs=None, penalty='l2', random_state=None, solver='lbfgs', tol=0.0001, verbose=0, warm_start=False), n_jobs=None, pre_dispatch='2*n_jobs', random_state=42, shuffle=True, store_train_meta_features=False, stratify=True, use_clones=True, use_features_in_secondary=False, use_probas=False, verbose=0)","title":"Example 4 - Stacking of Classifiers that Operate on Different Feature Subsets"},{"location":"user_guide/classifier/StackingCVClassifier/#example-5-roc-curve-with-decision_function","text":"Like other scikit-learn classifiers, the StackingCVClassifier has an decision_function method that can be used for plotting ROC curves. Note that the decision_function expects and requires the meta-classifier to implement a decision_function . from sklearn import model_selection from sklearn.linear_model import LogisticRegression from sklearn.neighbors import KNeighborsClassifier from sklearn.svm import SVC from sklearn.ensemble import RandomForestClassifier from mlxtend.classifier import StackingCVClassifier from sklearn.metrics import roc_curve, auc import numpy as np from sklearn.model_selection import train_test_split from sklearn import datasets from sklearn.preprocessing import label_binarize from sklearn.multiclass import OneVsRestClassifier iris = datasets.load_iris() X, y = iris.data[:, [0, 1]], iris.target # Binarize the output y = label_binarize(y, classes=[0, 1, 2]) n_classes = y.shape[1] RANDOM_SEED = 42 X_train, X_test, y_train, y_test = train_test_split( X, y, test_size=0.33, random_state=RANDOM_SEED) clf1 = LogisticRegression() clf2 = RandomForestClassifier(random_state=RANDOM_SEED) clf3 = SVC(random_state=RANDOM_SEED) lr = LogisticRegression() sclf = StackingCVClassifier(classifiers=[clf1, clf2, clf3], meta_classifier=lr) # Learn to predict each class against the other classifier = OneVsRestClassifier(sclf) Using predict_proba() y_score = classifier.fit(X_train, y_train).predict_proba(X_test) # Compute ROC curve and ROC area for each class fpr = dict() tpr = dict() roc_auc = dict() for i in range(n_classes): fpr[i], tpr[i], _ = roc_curve(y_test[:, i], y_score[:, i]) roc_auc[i] = auc(fpr[i], tpr[i]) # Compute micro-average ROC curve and ROC area fpr[\"micro\"], tpr[\"micro\"], _ = roc_curve(y_test.ravel(), y_score.ravel()) roc_auc[\"micro\"] = auc(fpr[\"micro\"], tpr[\"micro\"]) plt.figure() lw = 2 plt.plot(fpr[2], tpr[2], color='darkorange', lw=lw, label='ROC curve (area = %0.2f)' % roc_auc[2]) plt.plot([0, 1], [0, 1], color='navy', lw=lw, linestyle='--') plt.xlim([0.0, 1.0]) plt.ylim([0.0, 1.05]) plt.xlabel('False Positive Rate') plt.ylabel('True Positive Rate') plt.title('Receiver operating characteristic example') plt.legend(loc=\"lower right\") plt.show() Using decision_function() y_score = classifier.fit(X_train, y_train).decision_function(X_test) # Compute ROC curve and ROC area for each class fpr = dict() tpr = dict() roc_auc = dict() for i in range(n_classes): fpr[i], tpr[i], _ = roc_curve(y_test[:, i], y_score[:, i]) roc_auc[i] = auc(fpr[i], tpr[i]) # Compute micro-average ROC curve and ROC area fpr[\"micro\"], tpr[\"micro\"], _ = roc_curve(y_test.ravel(), y_score.ravel()) roc_auc[\"micro\"] = auc(fpr[\"micro\"], tpr[\"micro\"]) plt.figure() lw = 2 plt.plot(fpr[2], tpr[2], color='darkorange', lw=lw, label='ROC curve (area = %0.2f)' % roc_auc[2]) plt.plot([0, 1], [0, 1], color='navy', lw=lw, linestyle='--') plt.xlim([0.0, 1.0]) plt.ylim([0.0, 1.05]) plt.xlabel('False Positive Rate') plt.ylabel('True Positive Rate') plt.title('Receiver operating characteristic example') plt.legend(loc=\"lower right\") plt.show()","title":"Example 5 -- ROC Curve with decision_function"},{"location":"user_guide/classifier/StackingCVClassifier/#api","text":"StackingCVClassifier(classifiers, meta_classifier, use_probas=False, drop_proba_col=None, cv=2, shuffle=True, random_state=None, stratify=True, verbose=0, use_features_in_secondary=False, store_train_meta_features=False, use_clones=True, n_jobs=None, pre_dispatch='2 n_jobs')* A 'Stacking Cross-Validation' classifier for scikit-learn estimators. New in mlxtend v0.4.3 Parameters classifiers : array-like, shape = [n_classifiers] A list of classifiers. Invoking the fit method on the StackingCVClassifer will fit clones of these original classifiers that will be stored in the class attribute self.clfs_ . meta_classifier : object The meta-classifier to be fitted on the ensemble of classifiers use_probas : bool (default: False) If True, trains meta-classifier based on predicted probabilities instead of class labels. drop_proba_col : string (default: None) Drops extra \"probability\" column in the feature set, because it is redundant: p(y_c) = 1 - p(y_1) + p(y_2) + ... + p(y_{c-1}). This can be useful for meta-classifiers that are sensitive to perfectly collinear features. If last , drops last probability column. If first , drops first probability column. Only relevant if use_probas=True . cv : int, cross-validation generator or an iterable, optional (default: 2) Determines the cross-validation splitting strategy. Possible inputs for cv are: - None, to use the default 2-fold cross validation, - integer, to specify the number of folds in a (Stratified)KFold , - An object to be used as a cross-validation generator. - An iterable yielding train, test splits. For integer/None inputs, it will use either a KFold or StratifiedKFold cross validation depending the value of stratify argument. shuffle : bool (default: True) If True, and the cv argument is integer, the training data will be shuffled at fitting stage prior to cross-validation. If the cv argument is a specific cross validation technique, this argument is omitted. random_state : int, RandomState instance or None, optional (default: None) Constrols the randomness of the cv splitter. Used when cv is integer and shuffle=True . New in v0.16.0. stratify : bool (default: True) If True, and the cv argument is integer it will follow a stratified K-Fold cross validation technique. If the cv argument is a specific cross validation technique, this argument is omitted. verbose : int, optional (default=0) Controls the verbosity of the building process. - verbose=0 (default): Prints nothing - verbose=1 : Prints the number & name of the regressor being fitted and which fold is currently being used for fitting - verbose=2 : Prints info about the parameters of the regressor being fitted - verbose>2 : Changes verbose param of the underlying regressor to self.verbose - 2 use_features_in_secondary : bool (default: False) If True, the meta-classifier will be trained both on the predictions of the original classifiers and the original dataset. If False, the meta-classifier will be trained only on the predictions of the original classifiers. store_train_meta_features : bool (default: False) If True, the meta-features computed from the training data used for fitting the meta-classifier stored in the self.train_meta_features_ array, which can be accessed after calling fit . use_clones : bool (default: True) Clones the classifiers for stacking classification if True (default) or else uses the original ones, which will be refitted on the dataset upon calling the fit method. Hence, if use_clones=True, the original input classifiers will remain unmodified upon using the StackingCVClassifier's fit method. Setting use_clones=False is recommended if you are working with estimators that are supporting the scikit-learn fit/predict API interface but are not compatible to scikit-learn's clone function. n_jobs : int or None, optional (default=None) The number of CPUs to use to do the computation. None means 1 unless in a :obj: joblib.parallel_backend context. -1 means using all processors. See :term: Glossary <n_jobs> for more details. New in v0.16.0. pre_dispatch : int, or string, optional Controls the number of jobs that get dispatched during parallel execution. Reducing this number can be useful to avoid an explosion of memory consumption when more jobs get dispatched than CPUs can process. This parameter can be: - None, in which case all the jobs are immediately created and spawned. Use this for lightweight and fast-running jobs, to avoid delays due to on-demand spawning of the jobs - An int, giving the exact number of total jobs that are spawned - A string, giving an expression as a function of n_jobs, as in '2*n_jobs' New in v0.16.0. Attributes clfs_ : list, shape=[n_classifiers] Fitted classifiers (clones of the original classifiers) meta_clf_ : estimator Fitted meta-classifier (clone of the original meta-estimator) train_meta_features : numpy array, shape = [n_samples, n_classifiers] meta-features for training data, where n_samples is the number of samples in training data and n_classifiers is the number of classfiers. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/classifier/StackingCVClassifier/","title":"API"},{"location":"user_guide/classifier/StackingCVClassifier/#methods","text":"decision_function(X) Predict class confidence scores for X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns scores : shape=(n_samples,) if n_classes == 2 else (n_samples, n_classes). Confidence scores per (sample, class) combination. In the binary case, confidence score for self.classes_[1] where >0 means this class would be predicted. fit(X, y, groups=None, sample_weight=None) Fit ensemble classifers and the meta-classifier. Parameters X : numpy array, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : numpy array, shape = [n_samples] Target values. groups : numpy array/None, shape = [n_samples] The group that each sample belongs to. This is used by specific folding strategies such as GroupKFold() sample_weight : array-like, shape = [n_samples], optional Sample weights passed as sample_weights to each regressor in the regressors list as well as the meta_regressor. Raises error if some regressor does not support sample_weight in the fit() method. Returns self : object fit_transform(X, y=None, fit_params) Fit to data, then transform it. Fits transformer to X and y with optional parameters fit_params and returns a transformed version of X. Parameters X : numpy array of shape [n_samples, n_features] Training set. y : numpy array of shape [n_samples] Target values. **fit_params : dict Additional fit parameters. Returns X_new : numpy array of shape [n_samples, n_features_new] Transformed array. get_params(deep=True) Return estimator parameter names for GridSearch support. predict(X) Predict target values for X. Parameters X : numpy array, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns labels : array-like, shape = [n_samples] Predicted class labels. predict_meta_features(X) Get meta-features of test-data. Parameters X : numpy array, shape = [n_samples, n_features] Test vectors, where n_samples is the number of samples and n_features is the number of features. Returns meta-features : numpy array, shape = [n_samples, n_classifiers] Returns the meta-features for test data. predict_proba(X) Predict class probabilities for X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns proba : array-like, shape = [n_samples, n_classes] or a list of n_outputs of such arrays if n_outputs > 1. Probability for each class per sample. score(X, y, sample_weight=None) Return the mean accuracy on the given test data and labels. In multi-label classification, this is the subset accuracy which is a harsh metric since you require for each sample that each label set be correctly predicted. Parameters X : array-like of shape (n_samples, n_features) Test samples. y : array-like of shape (n_samples,) or (n_samples, n_outputs) True labels for X. sample_weight : array-like of shape (n_samples,), default=None Sample weights. Returns score : float Mean accuracy of self.predict(X) wrt. y. set_params( params) Set the parameters of this estimator. Valid parameter keys can be listed with get_params() . Returns self","title":"Methods"},{"location":"user_guide/classifier/StackingCVClassifier/#properties","text":"named_classifiers None","title":"Properties"},{"location":"user_guide/classifier/StackingClassifier/","text":"StackingClassifier: Simple stacking An ensemble-learning meta-classifier for stacking. from mlxtend.classifier import StackingClassifier Overview Stacking is an ensemble learning technique to combine multiple classification models via a meta-classifier. The individual classification models are trained based on the complete training set; then, the meta-classifier is fitted based on the outputs -- meta-features -- of the individual classification models in the ensemble. The meta-classifier can either be trained on the predicted class labels or probabilities from the ensemble. The algorithm can be summarized as follows (source: [1]): Please note that this type of Stacking is prone to overfitting due to information leakage. The related StackingCVClassifier.md does not derive the predictions for the 2nd-level classifier from the same datast that was used for training the level-1 classifiers and is recommended instead. References [1] Tang, J., S. Alelyani, and H. Liu. \" Data Classification: Algorithms and Applications. \" Data Mining and Knowledge Discovery Series, CRC Press (2015): pp. 498-500. [2] Wolpert, David H. \" Stacked generalization. \" Neural networks 5.2 (1992): 241-259. [3] Marios Michailidis (2017), StackNet, StackNet Meta Modelling Framework, https://github.com/kaz-Anova/StackNet Example 1 - Simple Stacked Classification from sklearn import datasets iris = datasets.load_iris() X, y = iris.data[:, 1:3], iris.target from sklearn import model_selection from sklearn.linear_model import LogisticRegression from sklearn.neighbors import KNeighborsClassifier from sklearn.naive_bayes import GaussianNB from sklearn.ensemble import RandomForestClassifier from mlxtend.classifier import StackingClassifier import numpy as np import warnings warnings.simplefilter('ignore') clf1 = KNeighborsClassifier(n_neighbors=1) clf2 = RandomForestClassifier(random_state=1) clf3 = GaussianNB() lr = LogisticRegression() sclf = StackingClassifier(classifiers=[clf1, clf2, clf3], meta_classifier=lr) print('3-fold cross validation:\\n') for clf, label in zip([clf1, clf2, clf3, sclf], ['KNN', 'Random Forest', 'Naive Bayes', 'StackingClassifier']): scores = model_selection.cross_val_score(clf, X, y, cv=3, scoring='accuracy') print(\"Accuracy: %0.2f (+/- %0.2f) [%s]\" % (scores.mean(), scores.std(), label)) 3-fold cross validation: Accuracy: 0.91 (+/- 0.01) [KNN] Accuracy: 0.95 (+/- 0.01) [Random Forest] Accuracy: 0.91 (+/- 0.02) [Naive Bayes] Accuracy: 0.95 (+/- 0.02) [StackingClassifier] import matplotlib.pyplot as plt from mlxtend.plotting import plot_decision_regions import matplotlib.gridspec as gridspec import itertools gs = gridspec.GridSpec(2, 2) fig = plt.figure(figsize=(10,8)) for clf, lab, grd in zip([clf1, clf2, clf3, sclf], ['KNN', 'Random Forest', 'Naive Bayes', 'StackingClassifier'], itertools.product([0, 1], repeat=2)): clf.fit(X, y) ax = plt.subplot(gs[grd[0], grd[1]]) fig = plot_decision_regions(X=X, y=y, clf=clf) plt.title(lab) Example 2 - Using Probabilities as Meta-Features Alternatively, the class-probabilities of the first-level classifiers can be used to train the meta-classifier (2nd-level classifier) by setting use_probas=True . If average_probas=True , the probabilities of the level-1 classifiers are averaged, if average_probas=False , the probabilities are stacked (recommended). For example, in a 3-class setting with 2 level-1 classifiers, these classifiers may make the following \"probability\" predictions for 1 training sample: classifier 1: [0.2, 0.5, 0.3] classifier 2: [0.3, 0.4, 0.4] If average_probas=True , the meta-features would be: [0.25, 0.45, 0.35] In contrast, using average_probas=False results in k features where, k = [n_classes * n_classifiers], by stacking these level-1 probabilities: [0.2, 0.5, 0.3, 0.3, 0.4, 0.4] clf1 = KNeighborsClassifier(n_neighbors=1) clf2 = RandomForestClassifier(random_state=1) clf3 = GaussianNB() lr = LogisticRegression() sclf = StackingClassifier(classifiers=[clf1, clf2, clf3], use_probas=True, average_probas=False, meta_classifier=lr) print('3-fold cross validation:\\n') for clf, label in zip([clf1, clf2, clf3, sclf], ['KNN', 'Random Forest', 'Naive Bayes', 'StackingClassifier']): scores = model_selection.cross_val_score(clf, X, y, cv=3, scoring='accuracy') print(\"Accuracy: %0.2f (+/- %0.2f) [%s]\" % (scores.mean(), scores.std(), label)) 3-fold cross validation: Accuracy: 0.91 (+/- 0.01) [KNN] Accuracy: 0.95 (+/- 0.01) [Random Forest] Accuracy: 0.91 (+/- 0.02) [Naive Bayes] Accuracy: 0.92 (+/- 0.02) [StackingClassifier] Example 3 - Stacked Classification and GridSearch The stack allows tuning hyper parameters of the base and meta models! A full list of tunable parameters can be obtained via estimator.get_params().keys() . from sklearn.linear_model import LogisticRegression from sklearn.neighbors import KNeighborsClassifier from sklearn.naive_bayes import GaussianNB from sklearn.ensemble import RandomForestClassifier from sklearn.model_selection import GridSearchCV from mlxtend.classifier import StackingClassifier # Initializing models clf1 = KNeighborsClassifier(n_neighbors=1) clf2 = RandomForestClassifier(random_state=1) clf3 = GaussianNB() lr = LogisticRegression() sclf = StackingClassifier(classifiers=[clf1, clf2, clf3], meta_classifier=lr) params = {'kneighborsclassifier__n_neighbors': [1, 5], 'randomforestclassifier__n_estimators': [10, 50], 'meta_classifier__C': [0.1, 10.0]} grid = GridSearchCV(estimator=sclf, param_grid=params, cv=5, refit=True) grid.fit(X, y) cv_keys = ('mean_test_score', 'std_test_score', 'params') for r, _ in enumerate(grid.cv_results_['mean_test_score']): print(\"%0.3f +/- %0.2f %r\" % (grid.cv_results_[cv_keys[0]][r], grid.cv_results_[cv_keys[1]][r] / 2.0, grid.cv_results_[cv_keys[2]][r])) print('Best parameters: %s' % grid.best_params_) print('Accuracy: %.2f' % grid.best_score_) 0.933 +/- 0.03 {'kneighborsclassifier__n_neighbors': 1, 'meta_classifier__C': 0.1, 'randomforestclassifier__n_estimators': 10} 0.940 +/- 0.02 {'kneighborsclassifier__n_neighbors': 1, 'meta_classifier__C': 0.1, 'randomforestclassifier__n_estimators': 50} 0.927 +/- 0.03 {'kneighborsclassifier__n_neighbors': 1, 'meta_classifier__C': 10.0, 'randomforestclassifier__n_estimators': 10} 0.947 +/- 0.02 {'kneighborsclassifier__n_neighbors': 1, 'meta_classifier__C': 10.0, 'randomforestclassifier__n_estimators': 50} 0.947 +/- 0.02 {'kneighborsclassifier__n_neighbors': 5, 'meta_classifier__C': 0.1, 'randomforestclassifier__n_estimators': 10} 0.947 +/- 0.02 {'kneighborsclassifier__n_neighbors': 5, 'meta_classifier__C': 0.1, 'randomforestclassifier__n_estimators': 50} 0.933 +/- 0.02 {'kneighborsclassifier__n_neighbors': 5, 'meta_classifier__C': 10.0, 'randomforestclassifier__n_estimators': 10} 0.940 +/- 0.02 {'kneighborsclassifier__n_neighbors': 5, 'meta_classifier__C': 10.0, 'randomforestclassifier__n_estimators': 50} Best parameters: {'kneighborsclassifier__n_neighbors': 1, 'meta_classifier__C': 10.0, 'randomforestclassifier__n_estimators': 50} Accuracy: 0.95 In case we are planning to use a regression algorithm multiple times, all we need to do is to add an additional number suffix in the parameter grid as shown below: from sklearn.model_selection import GridSearchCV # Initializing models clf1 = KNeighborsClassifier(n_neighbors=1) clf2 = RandomForestClassifier(random_state=1) clf3 = GaussianNB() lr = LogisticRegression() sclf = StackingClassifier(classifiers=[clf1, clf1, clf2, clf3], meta_classifier=lr) params = {'kneighborsclassifier-1__n_neighbors': [1, 5], 'kneighborsclassifier-2__n_neighbors': [1, 5], 'randomforestclassifier__n_estimators': [10, 50], 'meta_classifier__C': [0.1, 10.0]} grid = GridSearchCV(estimator=sclf, param_grid=params, cv=5, refit=True) grid.fit(X, y) cv_keys = ('mean_test_score', 'std_test_score', 'params') for r, _ in enumerate(grid.cv_results_['mean_test_score']): print(\"%0.3f +/- %0.2f %r\" % (grid.cv_results_[cv_keys[0]][r], grid.cv_results_[cv_keys[1]][r] / 2.0, grid.cv_results_[cv_keys[2]][r])) print('Best parameters: %s' % grid.best_params_) print('Accuracy: %.2f' % grid.best_score_) 0.933 +/- 0.03 {'kneighborsclassifier-1__n_neighbors': 1, 'kneighborsclassifier-2__n_neighbors': 1, 'meta_classifier__C': 0.1, 'randomforestclassifier__n_estimators': 10} 0.940 +/- 0.02 {'kneighborsclassifier-1__n_neighbors': 1, 'kneighborsclassifier-2__n_neighbors': 1, 'meta_classifier__C': 0.1, 'randomforestclassifier__n_estimators': 50} 0.927 +/- 0.03 {'kneighborsclassifier-1__n_neighbors': 1, 'kneighborsclassifier-2__n_neighbors': 1, 'meta_classifier__C': 10.0, 'randomforestclassifier__n_estimators': 10} 0.947 +/- 0.02 {'kneighborsclassifier-1__n_neighbors': 1, 'kneighborsclassifier-2__n_neighbors': 1, 'meta_classifier__C': 10.0, 'randomforestclassifier__n_estimators': 50} 0.940 +/- 0.02 {'kneighborsclassifier-1__n_neighbors': 1, 'kneighborsclassifier-2__n_neighbors': 5, 'meta_classifier__C': 0.1, 'randomforestclassifier__n_estimators': 10} 0.947 +/- 0.02 {'kneighborsclassifier-1__n_neighbors': 1, 'kneighborsclassifier-2__n_neighbors': 5, 'meta_classifier__C': 0.1, 'randomforestclassifier__n_estimators': 50} 0.933 +/- 0.03 {'kneighborsclassifier-1__n_neighbors': 1, 'kneighborsclassifier-2__n_neighbors': 5, 'meta_classifier__C': 10.0, 'randomforestclassifier__n_estimators': 10} 0.953 +/- 0.02 {'kneighborsclassifier-1__n_neighbors': 1, 'kneighborsclassifier-2__n_neighbors': 5, 'meta_classifier__C': 10.0, 'randomforestclassifier__n_estimators': 50} 0.940 +/- 0.02 {'kneighborsclassifier-1__n_neighbors': 5, 'kneighborsclassifier-2__n_neighbors': 1, 'meta_classifier__C': 0.1, 'randomforestclassifier__n_estimators': 10} 0.947 +/- 0.02 {'kneighborsclassifier-1__n_neighbors': 5, 'kneighborsclassifier-2__n_neighbors': 1, 'meta_classifier__C': 0.1, 'randomforestclassifier__n_estimators': 50} 0.933 +/- 0.03 {'kneighborsclassifier-1__n_neighbors': 5, 'kneighborsclassifier-2__n_neighbors': 1, 'meta_classifier__C': 10.0, 'randomforestclassifier__n_estimators': 10} 0.953 +/- 0.02 {'kneighborsclassifier-1__n_neighbors': 5, 'kneighborsclassifier-2__n_neighbors': 1, 'meta_classifier__C': 10.0, 'randomforestclassifier__n_estimators': 50} 0.953 +/- 0.02 {'kneighborsclassifier-1__n_neighbors': 5, 'kneighborsclassifier-2__n_neighbors': 5, 'meta_classifier__C': 0.1, 'randomforestclassifier__n_estimators': 10} 0.953 +/- 0.02 {'kneighborsclassifier-1__n_neighbors': 5, 'kneighborsclassifier-2__n_neighbors': 5, 'meta_classifier__C': 0.1, 'randomforestclassifier__n_estimators': 50} 0.933 +/- 0.02 {'kneighborsclassifier-1__n_neighbors': 5, 'kneighborsclassifier-2__n_neighbors': 5, 'meta_classifier__C': 10.0, 'randomforestclassifier__n_estimators': 10} 0.940 +/- 0.02 {'kneighborsclassifier-1__n_neighbors': 5, 'kneighborsclassifier-2__n_neighbors': 5, 'meta_classifier__C': 10.0, 'randomforestclassifier__n_estimators': 50} Best parameters: {'kneighborsclassifier-1__n_neighbors': 1, 'kneighborsclassifier-2__n_neighbors': 5, 'meta_classifier__C': 10.0, 'randomforestclassifier__n_estimators': 50} Accuracy: 0.95 Note The StackingClassifier also enables grid search over the classifiers argument. When there are level-mixed hyperparameters, GridSearchCV will try to replace hyperparameters in a top-down order, i.e., classifers -> single base classifier -> classifier hyperparameter. For instance, given a hyperparameter grid such as params = {'randomforestclassifier__n_estimators': [1, 100], 'classifiers': [(clf1, clf1, clf1), (clf2, clf3)]} it will first use the instance settings of either (clf1, clf1, clf1) or (clf2, clf3). Then it will replace the 'n_estimators' settings for a matching classifier based on 'randomforestclassifier__n_estimators': [1, 100] . Example 4 - Stacking of Classifiers that Operate on Different Feature Subsets The different level-1 classifiers can be fit to different subsets of features in the training dataset. The following example illustrates how this can be done on a technical level using scikit-learn pipelines and the ColumnSelector : from sklearn.datasets import load_iris from mlxtend.classifier import StackingClassifier from mlxtend.feature_selection import ColumnSelector from sklearn.pipeline import make_pipeline from sklearn.linear_model import LogisticRegression iris = load_iris() X = iris.data y = iris.target pipe1 = make_pipeline(ColumnSelector(cols=(0, 2)), LogisticRegression()) pipe2 = make_pipeline(ColumnSelector(cols=(1, 2, 3)), LogisticRegression()) sclf = StackingClassifier(classifiers=[pipe1, pipe2], meta_classifier=LogisticRegression()) sclf.fit(X, y) StackingClassifier(average_probas=False, classifiers=[Pipeline(memory=None, steps=[('columnselector', ColumnSelector(cols=(0, 2), drop_axis=False)), ('logisticregression', LogisticRegression(C=1.0, class_weight=None, dual=False, fit_intercept=True, intercept_scaling=1, l1_ratio=None, max_iter=100, multi_class='auto', n_jobs=None, penalty='l2', random_state=None, sol... meta_classifier=LogisticRegression(C=1.0, class_weight=None, dual=False, fit_intercept=True, intercept_scaling=1, l1_ratio=None, max_iter=100, multi_class='auto', n_jobs=None, penalty='l2', random_state=None, solver='lbfgs', tol=0.0001, verbose=0, warm_start=False), store_train_meta_features=False, use_clones=True, use_features_in_secondary=False, use_probas=False, verbose=0) Example 5 - Using Pre-fitted Classifiers Assume that we previously fitted our classifiers: from sklearn import model_selection from sklearn.linear_model import LogisticRegression from sklearn.naive_bayes import GaussianNB from sklearn.ensemble import RandomForestClassifier import numpy as np clf1 = KNeighborsClassifier(n_neighbors=1) clf2 = RandomForestClassifier(random_state=1) clf3 = GaussianNB() lr = LogisticRegression() for clf in (clf1, clf2, clf3): clf.fit(X, y) By setting fit_base_estimators=False , it will enforce use_clones to be False and the StackingClassifier will not re-fit these classifers to save computational time: from mlxtend.classifier import StackingClassifier import copy sclf = StackingClassifier(classifiers=[clf1, clf2, clf3], meta_classifier=lr, fit_base_estimators=False) labels = ['KNN', 'Random Forest', 'Naive Bayes', 'StackingClassifier'] sclf.fit(X, y) print('accuracy:', np.mean(y == sclf.predict(X))) Warning: enforce use_clones to be False accuracy: 1.0 However, please note that fit_base_estimators=False is incompatible to any form of cross-validation that is done in e.g., model_selection.cross_val_score or model_selection.GridSearchCV , etc., since it would require the classifiers to be refit to the training folds. Thus, only use fit_base_estimators=False if you want to make a prediction directly without cross-validation. Example 6 -- ROC Curve with decision_function Like other scikit-learn classifiers, the StackingCVClassifier has an decision_function method that can be used for plotting ROC curves. Note that the decision_function expects and requires the meta-classifier to implement a decision_function . from sklearn import model_selection from sklearn.linear_model import LogisticRegression from sklearn.neighbors import KNeighborsClassifier from sklearn.svm import SVC from sklearn.ensemble import RandomForestClassifier from mlxtend.classifier import StackingCVClassifier from sklearn.metrics import roc_curve, auc import numpy as np from sklearn.model_selection import train_test_split from sklearn import datasets from sklearn.preprocessing import label_binarize from sklearn.multiclass import OneVsRestClassifier iris = datasets.load_iris() X, y = iris.data[:, [0, 1]], iris.target # Binarize the output y = label_binarize(y, classes=[0, 1, 2]) n_classes = y.shape[1] RANDOM_SEED = 42 X_train, X_test, y_train, y_test = train_test_split( X, y, test_size=0.33, random_state=RANDOM_SEED) clf1 = LogisticRegression() clf2 = RandomForestClassifier(random_state=RANDOM_SEED) clf3 = SVC(random_state=RANDOM_SEED) lr = LogisticRegression() sclf = StackingClassifier(classifiers=[clf1, clf2, clf3], meta_classifier=lr) # Learn to predict each class against the other classifier = OneVsRestClassifier(sclf) Using predict_proba() y_score = classifier.fit(X_train, y_train).predict_proba(X_test) # Compute ROC curve and ROC area for each class fpr = dict() tpr = dict() roc_auc = dict() for i in range(n_classes): fpr[i], tpr[i], _ = roc_curve(y_test[:, i], y_score[:, i]) roc_auc[i] = auc(fpr[i], tpr[i]) # Compute micro-average ROC curve and ROC area fpr[\"micro\"], tpr[\"micro\"], _ = roc_curve(y_test.ravel(), y_score.ravel()) roc_auc[\"micro\"] = auc(fpr[\"micro\"], tpr[\"micro\"]) plt.figure() lw = 2 plt.plot(fpr[2], tpr[2], color='darkorange', lw=lw, label='ROC curve (area = %0.2f)' % roc_auc[2]) plt.plot([0, 1], [0, 1], color='navy', lw=lw, linestyle='--') plt.xlim([0.0, 1.0]) plt.ylim([0.0, 1.05]) plt.xlabel('False Positive Rate') plt.ylabel('True Positive Rate') plt.title('Receiver operating characteristic example') plt.legend(loc=\"lower right\") plt.show() Using decision_function() y_score = classifier.fit(X_train, y_train).decision_function(X_test) # Compute ROC curve and ROC area for each class fpr = dict() tpr = dict() roc_auc = dict() for i in range(n_classes): fpr[i], tpr[i], _ = roc_curve(y_test[:, i], y_score[:, i]) roc_auc[i] = auc(fpr[i], tpr[i]) # Compute micro-average ROC curve and ROC area fpr[\"micro\"], tpr[\"micro\"], _ = roc_curve(y_test.ravel(), y_score.ravel()) roc_auc[\"micro\"] = auc(fpr[\"micro\"], tpr[\"micro\"]) plt.figure() lw = 2 plt.plot(fpr[2], tpr[2], color='darkorange', lw=lw, label='ROC curve (area = %0.2f)' % roc_auc[2]) plt.plot([0, 1], [0, 1], color='navy', lw=lw, linestyle='--') plt.xlim([0.0, 1.0]) plt.ylim([0.0, 1.05]) plt.xlabel('False Positive Rate') plt.ylabel('True Positive Rate') plt.title('Receiver operating characteristic example') plt.legend(loc=\"lower right\") plt.show() API","title":"StackingClassifier: Simple stacking"},{"location":"user_guide/classifier/StackingClassifier/#stackingclassifier-simple-stacking","text":"An ensemble-learning meta-classifier for stacking. from mlxtend.classifier import StackingClassifier","title":"StackingClassifier: Simple stacking"},{"location":"user_guide/classifier/StackingClassifier/#overview","text":"Stacking is an ensemble learning technique to combine multiple classification models via a meta-classifier. The individual classification models are trained based on the complete training set; then, the meta-classifier is fitted based on the outputs -- meta-features -- of the individual classification models in the ensemble. The meta-classifier can either be trained on the predicted class labels or probabilities from the ensemble. The algorithm can be summarized as follows (source: [1]): Please note that this type of Stacking is prone to overfitting due to information leakage. The related StackingCVClassifier.md does not derive the predictions for the 2nd-level classifier from the same datast that was used for training the level-1 classifiers and is recommended instead.","title":"Overview"},{"location":"user_guide/classifier/StackingClassifier/#references","text":"[1] Tang, J., S. Alelyani, and H. Liu. \" Data Classification: Algorithms and Applications. \" Data Mining and Knowledge Discovery Series, CRC Press (2015): pp. 498-500. [2] Wolpert, David H. \" Stacked generalization. \" Neural networks 5.2 (1992): 241-259. [3] Marios Michailidis (2017), StackNet, StackNet Meta Modelling Framework, https://github.com/kaz-Anova/StackNet","title":"References"},{"location":"user_guide/classifier/StackingClassifier/#example-1-simple-stacked-classification","text":"from sklearn import datasets iris = datasets.load_iris() X, y = iris.data[:, 1:3], iris.target from sklearn import model_selection from sklearn.linear_model import LogisticRegression from sklearn.neighbors import KNeighborsClassifier from sklearn.naive_bayes import GaussianNB from sklearn.ensemble import RandomForestClassifier from mlxtend.classifier import StackingClassifier import numpy as np import warnings warnings.simplefilter('ignore') clf1 = KNeighborsClassifier(n_neighbors=1) clf2 = RandomForestClassifier(random_state=1) clf3 = GaussianNB() lr = LogisticRegression() sclf = StackingClassifier(classifiers=[clf1, clf2, clf3], meta_classifier=lr) print('3-fold cross validation:\\n') for clf, label in zip([clf1, clf2, clf3, sclf], ['KNN', 'Random Forest', 'Naive Bayes', 'StackingClassifier']): scores = model_selection.cross_val_score(clf, X, y, cv=3, scoring='accuracy') print(\"Accuracy: %0.2f (+/- %0.2f) [%s]\" % (scores.mean(), scores.std(), label)) 3-fold cross validation: Accuracy: 0.91 (+/- 0.01) [KNN] Accuracy: 0.95 (+/- 0.01) [Random Forest] Accuracy: 0.91 (+/- 0.02) [Naive Bayes] Accuracy: 0.95 (+/- 0.02) [StackingClassifier] import matplotlib.pyplot as plt from mlxtend.plotting import plot_decision_regions import matplotlib.gridspec as gridspec import itertools gs = gridspec.GridSpec(2, 2) fig = plt.figure(figsize=(10,8)) for clf, lab, grd in zip([clf1, clf2, clf3, sclf], ['KNN', 'Random Forest', 'Naive Bayes', 'StackingClassifier'], itertools.product([0, 1], repeat=2)): clf.fit(X, y) ax = plt.subplot(gs[grd[0], grd[1]]) fig = plot_decision_regions(X=X, y=y, clf=clf) plt.title(lab)","title":"Example 1 - Simple Stacked Classification"},{"location":"user_guide/classifier/StackingClassifier/#example-2-using-probabilities-as-meta-features","text":"Alternatively, the class-probabilities of the first-level classifiers can be used to train the meta-classifier (2nd-level classifier) by setting use_probas=True . If average_probas=True , the probabilities of the level-1 classifiers are averaged, if average_probas=False , the probabilities are stacked (recommended). For example, in a 3-class setting with 2 level-1 classifiers, these classifiers may make the following \"probability\" predictions for 1 training sample: classifier 1: [0.2, 0.5, 0.3] classifier 2: [0.3, 0.4, 0.4] If average_probas=True , the meta-features would be: [0.25, 0.45, 0.35] In contrast, using average_probas=False results in k features where, k = [n_classes * n_classifiers], by stacking these level-1 probabilities: [0.2, 0.5, 0.3, 0.3, 0.4, 0.4] clf1 = KNeighborsClassifier(n_neighbors=1) clf2 = RandomForestClassifier(random_state=1) clf3 = GaussianNB() lr = LogisticRegression() sclf = StackingClassifier(classifiers=[clf1, clf2, clf3], use_probas=True, average_probas=False, meta_classifier=lr) print('3-fold cross validation:\\n') for clf, label in zip([clf1, clf2, clf3, sclf], ['KNN', 'Random Forest', 'Naive Bayes', 'StackingClassifier']): scores = model_selection.cross_val_score(clf, X, y, cv=3, scoring='accuracy') print(\"Accuracy: %0.2f (+/- %0.2f) [%s]\" % (scores.mean(), scores.std(), label)) 3-fold cross validation: Accuracy: 0.91 (+/- 0.01) [KNN] Accuracy: 0.95 (+/- 0.01) [Random Forest] Accuracy: 0.91 (+/- 0.02) [Naive Bayes] Accuracy: 0.92 (+/- 0.02) [StackingClassifier]","title":"Example 2 - Using Probabilities as Meta-Features"},{"location":"user_guide/classifier/StackingClassifier/#example-3-stacked-classification-and-gridsearch","text":"The stack allows tuning hyper parameters of the base and meta models! A full list of tunable parameters can be obtained via estimator.get_params().keys() . from sklearn.linear_model import LogisticRegression from sklearn.neighbors import KNeighborsClassifier from sklearn.naive_bayes import GaussianNB from sklearn.ensemble import RandomForestClassifier from sklearn.model_selection import GridSearchCV from mlxtend.classifier import StackingClassifier # Initializing models clf1 = KNeighborsClassifier(n_neighbors=1) clf2 = RandomForestClassifier(random_state=1) clf3 = GaussianNB() lr = LogisticRegression() sclf = StackingClassifier(classifiers=[clf1, clf2, clf3], meta_classifier=lr) params = {'kneighborsclassifier__n_neighbors': [1, 5], 'randomforestclassifier__n_estimators': [10, 50], 'meta_classifier__C': [0.1, 10.0]} grid = GridSearchCV(estimator=sclf, param_grid=params, cv=5, refit=True) grid.fit(X, y) cv_keys = ('mean_test_score', 'std_test_score', 'params') for r, _ in enumerate(grid.cv_results_['mean_test_score']): print(\"%0.3f +/- %0.2f %r\" % (grid.cv_results_[cv_keys[0]][r], grid.cv_results_[cv_keys[1]][r] / 2.0, grid.cv_results_[cv_keys[2]][r])) print('Best parameters: %s' % grid.best_params_) print('Accuracy: %.2f' % grid.best_score_) 0.933 +/- 0.03 {'kneighborsclassifier__n_neighbors': 1, 'meta_classifier__C': 0.1, 'randomforestclassifier__n_estimators': 10} 0.940 +/- 0.02 {'kneighborsclassifier__n_neighbors': 1, 'meta_classifier__C': 0.1, 'randomforestclassifier__n_estimators': 50} 0.927 +/- 0.03 {'kneighborsclassifier__n_neighbors': 1, 'meta_classifier__C': 10.0, 'randomforestclassifier__n_estimators': 10} 0.947 +/- 0.02 {'kneighborsclassifier__n_neighbors': 1, 'meta_classifier__C': 10.0, 'randomforestclassifier__n_estimators': 50} 0.947 +/- 0.02 {'kneighborsclassifier__n_neighbors': 5, 'meta_classifier__C': 0.1, 'randomforestclassifier__n_estimators': 10} 0.947 +/- 0.02 {'kneighborsclassifier__n_neighbors': 5, 'meta_classifier__C': 0.1, 'randomforestclassifier__n_estimators': 50} 0.933 +/- 0.02 {'kneighborsclassifier__n_neighbors': 5, 'meta_classifier__C': 10.0, 'randomforestclassifier__n_estimators': 10} 0.940 +/- 0.02 {'kneighborsclassifier__n_neighbors': 5, 'meta_classifier__C': 10.0, 'randomforestclassifier__n_estimators': 50} Best parameters: {'kneighborsclassifier__n_neighbors': 1, 'meta_classifier__C': 10.0, 'randomforestclassifier__n_estimators': 50} Accuracy: 0.95 In case we are planning to use a regression algorithm multiple times, all we need to do is to add an additional number suffix in the parameter grid as shown below: from sklearn.model_selection import GridSearchCV # Initializing models clf1 = KNeighborsClassifier(n_neighbors=1) clf2 = RandomForestClassifier(random_state=1) clf3 = GaussianNB() lr = LogisticRegression() sclf = StackingClassifier(classifiers=[clf1, clf1, clf2, clf3], meta_classifier=lr) params = {'kneighborsclassifier-1__n_neighbors': [1, 5], 'kneighborsclassifier-2__n_neighbors': [1, 5], 'randomforestclassifier__n_estimators': [10, 50], 'meta_classifier__C': [0.1, 10.0]} grid = GridSearchCV(estimator=sclf, param_grid=params, cv=5, refit=True) grid.fit(X, y) cv_keys = ('mean_test_score', 'std_test_score', 'params') for r, _ in enumerate(grid.cv_results_['mean_test_score']): print(\"%0.3f +/- %0.2f %r\" % (grid.cv_results_[cv_keys[0]][r], grid.cv_results_[cv_keys[1]][r] / 2.0, grid.cv_results_[cv_keys[2]][r])) print('Best parameters: %s' % grid.best_params_) print('Accuracy: %.2f' % grid.best_score_) 0.933 +/- 0.03 {'kneighborsclassifier-1__n_neighbors': 1, 'kneighborsclassifier-2__n_neighbors': 1, 'meta_classifier__C': 0.1, 'randomforestclassifier__n_estimators': 10} 0.940 +/- 0.02 {'kneighborsclassifier-1__n_neighbors': 1, 'kneighborsclassifier-2__n_neighbors': 1, 'meta_classifier__C': 0.1, 'randomforestclassifier__n_estimators': 50} 0.927 +/- 0.03 {'kneighborsclassifier-1__n_neighbors': 1, 'kneighborsclassifier-2__n_neighbors': 1, 'meta_classifier__C': 10.0, 'randomforestclassifier__n_estimators': 10} 0.947 +/- 0.02 {'kneighborsclassifier-1__n_neighbors': 1, 'kneighborsclassifier-2__n_neighbors': 1, 'meta_classifier__C': 10.0, 'randomforestclassifier__n_estimators': 50} 0.940 +/- 0.02 {'kneighborsclassifier-1__n_neighbors': 1, 'kneighborsclassifier-2__n_neighbors': 5, 'meta_classifier__C': 0.1, 'randomforestclassifier__n_estimators': 10} 0.947 +/- 0.02 {'kneighborsclassifier-1__n_neighbors': 1, 'kneighborsclassifier-2__n_neighbors': 5, 'meta_classifier__C': 0.1, 'randomforestclassifier__n_estimators': 50} 0.933 +/- 0.03 {'kneighborsclassifier-1__n_neighbors': 1, 'kneighborsclassifier-2__n_neighbors': 5, 'meta_classifier__C': 10.0, 'randomforestclassifier__n_estimators': 10} 0.953 +/- 0.02 {'kneighborsclassifier-1__n_neighbors': 1, 'kneighborsclassifier-2__n_neighbors': 5, 'meta_classifier__C': 10.0, 'randomforestclassifier__n_estimators': 50} 0.940 +/- 0.02 {'kneighborsclassifier-1__n_neighbors': 5, 'kneighborsclassifier-2__n_neighbors': 1, 'meta_classifier__C': 0.1, 'randomforestclassifier__n_estimators': 10} 0.947 +/- 0.02 {'kneighborsclassifier-1__n_neighbors': 5, 'kneighborsclassifier-2__n_neighbors': 1, 'meta_classifier__C': 0.1, 'randomforestclassifier__n_estimators': 50} 0.933 +/- 0.03 {'kneighborsclassifier-1__n_neighbors': 5, 'kneighborsclassifier-2__n_neighbors': 1, 'meta_classifier__C': 10.0, 'randomforestclassifier__n_estimators': 10} 0.953 +/- 0.02 {'kneighborsclassifier-1__n_neighbors': 5, 'kneighborsclassifier-2__n_neighbors': 1, 'meta_classifier__C': 10.0, 'randomforestclassifier__n_estimators': 50} 0.953 +/- 0.02 {'kneighborsclassifier-1__n_neighbors': 5, 'kneighborsclassifier-2__n_neighbors': 5, 'meta_classifier__C': 0.1, 'randomforestclassifier__n_estimators': 10} 0.953 +/- 0.02 {'kneighborsclassifier-1__n_neighbors': 5, 'kneighborsclassifier-2__n_neighbors': 5, 'meta_classifier__C': 0.1, 'randomforestclassifier__n_estimators': 50} 0.933 +/- 0.02 {'kneighborsclassifier-1__n_neighbors': 5, 'kneighborsclassifier-2__n_neighbors': 5, 'meta_classifier__C': 10.0, 'randomforestclassifier__n_estimators': 10} 0.940 +/- 0.02 {'kneighborsclassifier-1__n_neighbors': 5, 'kneighborsclassifier-2__n_neighbors': 5, 'meta_classifier__C': 10.0, 'randomforestclassifier__n_estimators': 50} Best parameters: {'kneighborsclassifier-1__n_neighbors': 1, 'kneighborsclassifier-2__n_neighbors': 5, 'meta_classifier__C': 10.0, 'randomforestclassifier__n_estimators': 50} Accuracy: 0.95 Note The StackingClassifier also enables grid search over the classifiers argument. When there are level-mixed hyperparameters, GridSearchCV will try to replace hyperparameters in a top-down order, i.e., classifers -> single base classifier -> classifier hyperparameter. For instance, given a hyperparameter grid such as params = {'randomforestclassifier__n_estimators': [1, 100], 'classifiers': [(clf1, clf1, clf1), (clf2, clf3)]} it will first use the instance settings of either (clf1, clf1, clf1) or (clf2, clf3). Then it will replace the 'n_estimators' settings for a matching classifier based on 'randomforestclassifier__n_estimators': [1, 100] .","title":"Example 3 - Stacked Classification and GridSearch"},{"location":"user_guide/classifier/StackingClassifier/#example-4-stacking-of-classifiers-that-operate-on-different-feature-subsets","text":"The different level-1 classifiers can be fit to different subsets of features in the training dataset. The following example illustrates how this can be done on a technical level using scikit-learn pipelines and the ColumnSelector : from sklearn.datasets import load_iris from mlxtend.classifier import StackingClassifier from mlxtend.feature_selection import ColumnSelector from sklearn.pipeline import make_pipeline from sklearn.linear_model import LogisticRegression iris = load_iris() X = iris.data y = iris.target pipe1 = make_pipeline(ColumnSelector(cols=(0, 2)), LogisticRegression()) pipe2 = make_pipeline(ColumnSelector(cols=(1, 2, 3)), LogisticRegression()) sclf = StackingClassifier(classifiers=[pipe1, pipe2], meta_classifier=LogisticRegression()) sclf.fit(X, y) StackingClassifier(average_probas=False, classifiers=[Pipeline(memory=None, steps=[('columnselector', ColumnSelector(cols=(0, 2), drop_axis=False)), ('logisticregression', LogisticRegression(C=1.0, class_weight=None, dual=False, fit_intercept=True, intercept_scaling=1, l1_ratio=None, max_iter=100, multi_class='auto', n_jobs=None, penalty='l2', random_state=None, sol... meta_classifier=LogisticRegression(C=1.0, class_weight=None, dual=False, fit_intercept=True, intercept_scaling=1, l1_ratio=None, max_iter=100, multi_class='auto', n_jobs=None, penalty='l2', random_state=None, solver='lbfgs', tol=0.0001, verbose=0, warm_start=False), store_train_meta_features=False, use_clones=True, use_features_in_secondary=False, use_probas=False, verbose=0)","title":"Example 4 - Stacking of Classifiers that Operate on Different Feature Subsets"},{"location":"user_guide/classifier/StackingClassifier/#example-5-using-pre-fitted-classifiers","text":"Assume that we previously fitted our classifiers: from sklearn import model_selection from sklearn.linear_model import LogisticRegression from sklearn.naive_bayes import GaussianNB from sklearn.ensemble import RandomForestClassifier import numpy as np clf1 = KNeighborsClassifier(n_neighbors=1) clf2 = RandomForestClassifier(random_state=1) clf3 = GaussianNB() lr = LogisticRegression() for clf in (clf1, clf2, clf3): clf.fit(X, y) By setting fit_base_estimators=False , it will enforce use_clones to be False and the StackingClassifier will not re-fit these classifers to save computational time: from mlxtend.classifier import StackingClassifier import copy sclf = StackingClassifier(classifiers=[clf1, clf2, clf3], meta_classifier=lr, fit_base_estimators=False) labels = ['KNN', 'Random Forest', 'Naive Bayes', 'StackingClassifier'] sclf.fit(X, y) print('accuracy:', np.mean(y == sclf.predict(X))) Warning: enforce use_clones to be False accuracy: 1.0 However, please note that fit_base_estimators=False is incompatible to any form of cross-validation that is done in e.g., model_selection.cross_val_score or model_selection.GridSearchCV , etc., since it would require the classifiers to be refit to the training folds. Thus, only use fit_base_estimators=False if you want to make a prediction directly without cross-validation.","title":"Example 5 - Using Pre-fitted Classifiers"},{"location":"user_guide/classifier/StackingClassifier/#example-6-roc-curve-with-decision_function","text":"Like other scikit-learn classifiers, the StackingCVClassifier has an decision_function method that can be used for plotting ROC curves. Note that the decision_function expects and requires the meta-classifier to implement a decision_function . from sklearn import model_selection from sklearn.linear_model import LogisticRegression from sklearn.neighbors import KNeighborsClassifier from sklearn.svm import SVC from sklearn.ensemble import RandomForestClassifier from mlxtend.classifier import StackingCVClassifier from sklearn.metrics import roc_curve, auc import numpy as np from sklearn.model_selection import train_test_split from sklearn import datasets from sklearn.preprocessing import label_binarize from sklearn.multiclass import OneVsRestClassifier iris = datasets.load_iris() X, y = iris.data[:, [0, 1]], iris.target # Binarize the output y = label_binarize(y, classes=[0, 1, 2]) n_classes = y.shape[1] RANDOM_SEED = 42 X_train, X_test, y_train, y_test = train_test_split( X, y, test_size=0.33, random_state=RANDOM_SEED) clf1 = LogisticRegression() clf2 = RandomForestClassifier(random_state=RANDOM_SEED) clf3 = SVC(random_state=RANDOM_SEED) lr = LogisticRegression() sclf = StackingClassifier(classifiers=[clf1, clf2, clf3], meta_classifier=lr) # Learn to predict each class against the other classifier = OneVsRestClassifier(sclf) Using predict_proba() y_score = classifier.fit(X_train, y_train).predict_proba(X_test) # Compute ROC curve and ROC area for each class fpr = dict() tpr = dict() roc_auc = dict() for i in range(n_classes): fpr[i], tpr[i], _ = roc_curve(y_test[:, i], y_score[:, i]) roc_auc[i] = auc(fpr[i], tpr[i]) # Compute micro-average ROC curve and ROC area fpr[\"micro\"], tpr[\"micro\"], _ = roc_curve(y_test.ravel(), y_score.ravel()) roc_auc[\"micro\"] = auc(fpr[\"micro\"], tpr[\"micro\"]) plt.figure() lw = 2 plt.plot(fpr[2], tpr[2], color='darkorange', lw=lw, label='ROC curve (area = %0.2f)' % roc_auc[2]) plt.plot([0, 1], [0, 1], color='navy', lw=lw, linestyle='--') plt.xlim([0.0, 1.0]) plt.ylim([0.0, 1.05]) plt.xlabel('False Positive Rate') plt.ylabel('True Positive Rate') plt.title('Receiver operating characteristic example') plt.legend(loc=\"lower right\") plt.show() Using decision_function() y_score = classifier.fit(X_train, y_train).decision_function(X_test) # Compute ROC curve and ROC area for each class fpr = dict() tpr = dict() roc_auc = dict() for i in range(n_classes): fpr[i], tpr[i], _ = roc_curve(y_test[:, i], y_score[:, i]) roc_auc[i] = auc(fpr[i], tpr[i]) # Compute micro-average ROC curve and ROC area fpr[\"micro\"], tpr[\"micro\"], _ = roc_curve(y_test.ravel(), y_score.ravel()) roc_auc[\"micro\"] = auc(fpr[\"micro\"], tpr[\"micro\"]) plt.figure() lw = 2 plt.plot(fpr[2], tpr[2], color='darkorange', lw=lw, label='ROC curve (area = %0.2f)' % roc_auc[2]) plt.plot([0, 1], [0, 1], color='navy', lw=lw, linestyle='--') plt.xlim([0.0, 1.0]) plt.ylim([0.0, 1.05]) plt.xlabel('False Positive Rate') plt.ylabel('True Positive Rate') plt.title('Receiver operating characteristic example') plt.legend(loc=\"lower right\") plt.show()","title":"Example 6 -- ROC Curve with decision_function"},{"location":"user_guide/classifier/StackingClassifier/#api","text":"","title":"API"},{"location":"user_guide/cluster/Kmeans/","text":"Kmeans: k-means clustering A implementation of k-means clustering. from mlxtend.cluster import Kmeans Overview Clustering falls into the category of unsupervised learning, a subfield of machine learning where the ground truth labels are not available to us in real-world applications. In clustering, our goal is to group samples by similarity (in k-means: Euclidean distance). The k-means algorithms can be summarized as follows: Randomly pick k centroids from the sample points as initial cluster centers. Assign each sample to the nearest centroid \\mu(j), \\; j \\in {1,...,k} . Move the centroids to the center of the samples that were assigned to it. Repeat steps 2 and 3 until the cluster assignments do not change or a user-defined tolerance or a maximum number of iterations is reached. References MacQueen, J. B. (1967). Some Methods for classification and Analysis of Multivariate Observations . Proceedings of 5th Berkeley Symposium on Mathematical Statistics and Probability. University of California Press. pp. 281\u2013297. MR 0214227. Zbl 0214.46201. Retrieved 2009-04-07. Example 1 - Three Blobs Load some sample data: import matplotlib.pyplot as plt from mlxtend.data import three_blobs_data X, y = three_blobs_data() plt.scatter(X[:, 0], X[:, 1], c='white') plt.show() Compute the cluster centroids: from mlxtend.cluster import Kmeans km = Kmeans(k=3, max_iter=50, random_seed=1, print_progress=3) km.fit(X) print('Iterations until convergence:', km.iterations_) print('Final centroids:\\n', km.centroids_) Iteration: 2/50 | Elapsed: 00:00:00 | ETA: 00:00:00 Iterations until convergence: 2 Final centroids: [[-1.5947298 2.92236966] [ 2.06521743 0.96137409] [ 0.9329651 4.35420713]] Visualize the cluster memberships: y_clust = km.predict(X) plt.scatter(X[y_clust == 0, 0], X[y_clust == 0, 1], s=50, c='lightgreen', marker='s', label='cluster 1') plt.scatter(X[y_clust == 1,0], X[y_clust == 1,1], s=50, c='orange', marker='o', label='cluster 2') plt.scatter(X[y_clust == 2,0], X[y_clust == 2,1], s=50, c='lightblue', marker='v', label='cluster 3') plt.scatter(km.centroids_[:,0], km.centroids_[:,1], s=250, marker='*', c='red', label='centroids') plt.legend(loc='lower left', scatterpoints=1) plt.grid() plt.show() API Kmeans(k, max_iter=10, convergence_tolerance=1e-05, random_seed=None, print_progress=0) K-means clustering class. Added in 0.4.1dev Parameters k : int Number of clusters max_iter : int (default: 10) Number of iterations during cluster assignment. Cluster re-assignment stops automatically when the algorithm converged. convergence_tolerance : float (default: 1e-05) Compares current centroids with centroids of the previous iteration using the given tolerance (a small positive float)to determine if the algorithm converged early. random_seed : int (default: None) Set random state for the initial centroid assignment. print_progress : int (default: 0) Prints progress in fitting to stderr. 0: No output 1: Iterations elapsed 2: 1 plus time elapsed 3: 2 plus estimated time until completion Attributes centroids_ : 2d-array, shape={k, n_features} Feature values of the k cluster centroids. custers_ : dictionary The cluster assignments stored as a Python dictionary; the dictionary keys denote the cluster indeces and the items are Python lists of the sample indices that were assigned to each cluster. iterations_ : int Number of iterations until convergence. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/classifier/Kmeans/ Methods fit(X, init_params=True) Learn model from training data. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. init_params : bool (default: True) Re-initializes model parameters prior to fitting. Set False to continue training with weights from a previous model fitting. Returns self : object predict(X) Predict targets from X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns target_values : array-like, shape = [n_samples] Predicted target values.","title":"Kmeans: k-means clustering"},{"location":"user_guide/cluster/Kmeans/#kmeans-k-means-clustering","text":"A implementation of k-means clustering. from mlxtend.cluster import Kmeans","title":"Kmeans: k-means clustering"},{"location":"user_guide/cluster/Kmeans/#overview","text":"Clustering falls into the category of unsupervised learning, a subfield of machine learning where the ground truth labels are not available to us in real-world applications. In clustering, our goal is to group samples by similarity (in k-means: Euclidean distance). The k-means algorithms can be summarized as follows: Randomly pick k centroids from the sample points as initial cluster centers. Assign each sample to the nearest centroid \\mu(j), \\; j \\in {1,...,k} . Move the centroids to the center of the samples that were assigned to it. Repeat steps 2 and 3 until the cluster assignments do not change or a user-defined tolerance or a maximum number of iterations is reached.","title":"Overview"},{"location":"user_guide/cluster/Kmeans/#references","text":"MacQueen, J. B. (1967). Some Methods for classification and Analysis of Multivariate Observations . Proceedings of 5th Berkeley Symposium on Mathematical Statistics and Probability. University of California Press. pp. 281\u2013297. MR 0214227. Zbl 0214.46201. Retrieved 2009-04-07.","title":"References"},{"location":"user_guide/cluster/Kmeans/#example-1-three-blobs","text":"","title":"Example 1 - Three Blobs"},{"location":"user_guide/cluster/Kmeans/#load-some-sample-data","text":"import matplotlib.pyplot as plt from mlxtend.data import three_blobs_data X, y = three_blobs_data() plt.scatter(X[:, 0], X[:, 1], c='white') plt.show()","title":"Load some sample data:"},{"location":"user_guide/cluster/Kmeans/#compute-the-cluster-centroids","text":"from mlxtend.cluster import Kmeans km = Kmeans(k=3, max_iter=50, random_seed=1, print_progress=3) km.fit(X) print('Iterations until convergence:', km.iterations_) print('Final centroids:\\n', km.centroids_) Iteration: 2/50 | Elapsed: 00:00:00 | ETA: 00:00:00 Iterations until convergence: 2 Final centroids: [[-1.5947298 2.92236966] [ 2.06521743 0.96137409] [ 0.9329651 4.35420713]]","title":"Compute the cluster centroids:"},{"location":"user_guide/cluster/Kmeans/#visualize-the-cluster-memberships","text":"y_clust = km.predict(X) plt.scatter(X[y_clust == 0, 0], X[y_clust == 0, 1], s=50, c='lightgreen', marker='s', label='cluster 1') plt.scatter(X[y_clust == 1,0], X[y_clust == 1,1], s=50, c='orange', marker='o', label='cluster 2') plt.scatter(X[y_clust == 2,0], X[y_clust == 2,1], s=50, c='lightblue', marker='v', label='cluster 3') plt.scatter(km.centroids_[:,0], km.centroids_[:,1], s=250, marker='*', c='red', label='centroids') plt.legend(loc='lower left', scatterpoints=1) plt.grid() plt.show()","title":"Visualize the cluster memberships:"},{"location":"user_guide/cluster/Kmeans/#api","text":"Kmeans(k, max_iter=10, convergence_tolerance=1e-05, random_seed=None, print_progress=0) K-means clustering class. Added in 0.4.1dev Parameters k : int Number of clusters max_iter : int (default: 10) Number of iterations during cluster assignment. Cluster re-assignment stops automatically when the algorithm converged. convergence_tolerance : float (default: 1e-05) Compares current centroids with centroids of the previous iteration using the given tolerance (a small positive float)to determine if the algorithm converged early. random_seed : int (default: None) Set random state for the initial centroid assignment. print_progress : int (default: 0) Prints progress in fitting to stderr. 0: No output 1: Iterations elapsed 2: 1 plus time elapsed 3: 2 plus estimated time until completion Attributes centroids_ : 2d-array, shape={k, n_features} Feature values of the k cluster centroids. custers_ : dictionary The cluster assignments stored as a Python dictionary; the dictionary keys denote the cluster indeces and the items are Python lists of the sample indices that were assigned to each cluster. iterations_ : int Number of iterations until convergence. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/classifier/Kmeans/","title":"API"},{"location":"user_guide/cluster/Kmeans/#methods","text":"fit(X, init_params=True) Learn model from training data. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. init_params : bool (default: True) Re-initializes model parameters prior to fitting. Set False to continue training with weights from a previous model fitting. Returns self : object predict(X) Predict targets from X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns target_values : array-like, shape = [n_samples] Predicted target values.","title":"Methods"},{"location":"user_guide/data/autompg_data/","text":"autompg_data: The Auto-MPG dataset for regression A function that loads the autompg dataset into NumPy arrays. from mlxtend.data import autompg_data Overview The Auto-MPG dataset for regression analysis. The target ( y ) is defined as the miles per gallon (mpg) for 392 automobiles (6 rows containing \"NaN\"s have been removed. The 8 feature columns are: Features cylinders: multi-valued discrete displacement: continuous horsepower: continuous weight: continuous acceleration: continuous model year: multi-valued discrete origin: multi-valued discrete car name: string (unique for each instance) Number of samples: 392 Target variable (continuous): mpg References Source: https://archive.ics.uci.edu/ml/datasets/Auto+MPG Quinlan,R. (1993). Combining Instance-Based and Model-Based Learning. In Proceedings on the Tenth International Conference of Machine Learning, 236-243, University of Massachusetts, Amherst. Morgan Kaufmann. Example - Dataset overview from mlxtend.data import autompg_data X, y = autompg_data() print('Dimensions: %s x %s' % (X.shape[0], X.shape[1])) print('\\nHeader: %s' % ['cylinders', 'displacement', 'horsepower', 'weight', 'acceleration', 'model year', 'origin', 'car name']) print('1st row', X[0]) Dimensions: 392 x 8 Header: ['cylinders', 'displacement', 'horsepower', 'weight', 'acceleration', 'model year', 'origin', 'car name'] 1st row [ 8.00000000e+00 3.07000000e+02 1.30000000e+02 3.50400000e+03 1.20000000e+01 7.00000000e+01 1.00000000e+00 nan] Note that the feature array contains a str column (\"car name\"), thus it is recommended to pick the features as needed and convert it into a float array for further analysis. The example below shows how to get rid of the car name column and cast the NumPy array as a float array. X[:, :-1].astype(float) array([[ 8. , 307. , 130. , ..., 12. , 70. , 1. ], [ 8. , 350. , 165. , ..., 11.5, 70. , 1. ], [ 8. , 318. , 150. , ..., 11. , 70. , 1. ], ..., [ 4. , 135. , 84. , ..., 11.6, 82. , 1. ], [ 4. , 120. , 79. , ..., 18.6, 82. , 1. ], [ 4. , 119. , 82. , ..., 19.4, 82. , 1. ]]) API autompg_data() Auto MPG dataset. Source : https://archive.ics.uci.edu/ml/datasets/Auto+MPG Number of samples : 392 Continuous target variable : mpg Dataset Attributes: 1) cylinders: multi-valued discrete 2) displacement: continuous 3) horsepower: continuous 4) weight: continuous 5) acceleration: continuous 6) model year: multi-valued discrete 7) origin: multi-valued discrete 8) car name: string (unique for each instance) Returns X, y : [n_samples, n_features], [n_targets] X is the feature matrix with 392 auto samples as rows and 8 feature columns (6 rows with NaNs removed). y is a 1-dimensional array of the target MPG values. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/data/autompg_data/","title":"autompg_data: The Auto-MPG dataset for regression"},{"location":"user_guide/data/autompg_data/#autompg_data-the-auto-mpg-dataset-for-regression","text":"A function that loads the autompg dataset into NumPy arrays. from mlxtend.data import autompg_data","title":"autompg_data: The Auto-MPG dataset for regression"},{"location":"user_guide/data/autompg_data/#overview","text":"The Auto-MPG dataset for regression analysis. The target ( y ) is defined as the miles per gallon (mpg) for 392 automobiles (6 rows containing \"NaN\"s have been removed. The 8 feature columns are: Features cylinders: multi-valued discrete displacement: continuous horsepower: continuous weight: continuous acceleration: continuous model year: multi-valued discrete origin: multi-valued discrete car name: string (unique for each instance) Number of samples: 392 Target variable (continuous): mpg","title":"Overview"},{"location":"user_guide/data/autompg_data/#references","text":"Source: https://archive.ics.uci.edu/ml/datasets/Auto+MPG Quinlan,R. (1993). Combining Instance-Based and Model-Based Learning. In Proceedings on the Tenth International Conference of Machine Learning, 236-243, University of Massachusetts, Amherst. Morgan Kaufmann.","title":"References"},{"location":"user_guide/data/autompg_data/#example-dataset-overview","text":"from mlxtend.data import autompg_data X, y = autompg_data() print('Dimensions: %s x %s' % (X.shape[0], X.shape[1])) print('\\nHeader: %s' % ['cylinders', 'displacement', 'horsepower', 'weight', 'acceleration', 'model year', 'origin', 'car name']) print('1st row', X[0]) Dimensions: 392 x 8 Header: ['cylinders', 'displacement', 'horsepower', 'weight', 'acceleration', 'model year', 'origin', 'car name'] 1st row [ 8.00000000e+00 3.07000000e+02 1.30000000e+02 3.50400000e+03 1.20000000e+01 7.00000000e+01 1.00000000e+00 nan] Note that the feature array contains a str column (\"car name\"), thus it is recommended to pick the features as needed and convert it into a float array for further analysis. The example below shows how to get rid of the car name column and cast the NumPy array as a float array. X[:, :-1].astype(float) array([[ 8. , 307. , 130. , ..., 12. , 70. , 1. ], [ 8. , 350. , 165. , ..., 11.5, 70. , 1. ], [ 8. , 318. , 150. , ..., 11. , 70. , 1. ], ..., [ 4. , 135. , 84. , ..., 11.6, 82. , 1. ], [ 4. , 120. , 79. , ..., 18.6, 82. , 1. ], [ 4. , 119. , 82. , ..., 19.4, 82. , 1. ]])","title":"Example - Dataset overview"},{"location":"user_guide/data/autompg_data/#api","text":"autompg_data() Auto MPG dataset. Source : https://archive.ics.uci.edu/ml/datasets/Auto+MPG Number of samples : 392 Continuous target variable : mpg Dataset Attributes: 1) cylinders: multi-valued discrete 2) displacement: continuous 3) horsepower: continuous 4) weight: continuous 5) acceleration: continuous 6) model year: multi-valued discrete 7) origin: multi-valued discrete 8) car name: string (unique for each instance) Returns X, y : [n_samples, n_features], [n_targets] X is the feature matrix with 392 auto samples as rows and 8 feature columns (6 rows with NaNs removed). y is a 1-dimensional array of the target MPG values. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/data/autompg_data/","title":"API"},{"location":"user_guide/data/boston_housing_data/","text":"boston_housing_data: The Boston housing dataset for regression A function that loads the boston_housing_data dataset into NumPy arrays. from mlxtend.data import boston_housing_data Overview The Boston Housing dataset for regression analysis. Features CRIM: per capita crime rate by town ZN: proportion of residential land zoned for lots over 25,000 sq.ft. INDUS: proportion of non-retail business acres per town CHAS: Charles River dummy variable (= 1 if tract bounds river; 0 otherwise) NOX: nitric oxides concentration (parts per 10 million) RM: average number of rooms per dwelling AGE: proportion of owner-occupied units built prior to 1940 DIS: weighted distances to five Boston employment centres RAD: index of accessibility to radial highways TAX: full-value property-tax rate per $10,000 PTRATIO: pupil-teacher ratio by town B: 1000(Bk - 0.63)^2 where Bk is the proportion of b. by town LSTAT: % lower status of the population Number of samples: 506 Target variable (continuous): MEDV, Median value of owner-occupied homes in $1000's References Source: https://archive.ics.uci.edu/ml/datasets/Wine Harrison, D. and Rubinfeld, D.L. 'Hedonic prices and the demand for clean air', J. Environ. Economics & Management, vol.5, 81-102, 1978. Example 1 - Dataset overview from mlxtend.data import boston_housing_data X, y = boston_housing_data() print('Dimensions: %s x %s' % (X.shape[0], X.shape[1])) print('1st row', X[0]) (506, 14) Dimensions: 506 x 13 1st row [ 6.32000000e-03 1.80000000e+01 2.31000000e+00 0.00000000e+00 5.38000000e-01 6.57500000e+00 6.52000000e+01 4.09000000e+00 1.00000000e+00 2.96000000e+02 1.53000000e+01 3.96900000e+02 4.98000000e+00] API boston_housing_data() Boston Housing dataset. Source : https://archive.ics.uci.edu/ml/datasets/Housing Number of samples : 506 Continuous target variable : MEDV MEDV = Median value of owner-occupied homes in $1000's Dataset Attributes: 1) CRIM per capita crime rate by town 2) ZN proportion of residential land zoned for lots over 25,000 sq.ft. 3) INDUS proportion of non-retail business acres per town 4) CHAS Charles River dummy variable (= 1 if tract bounds river; 0 otherwise) 5) NOX nitric oxides concentration (parts per 10 million) 6) RM average number of rooms per dwelling 7) AGE proportion of owner-occupied units built prior to 1940 8) DIS weighted distances to five Boston employment centres 9) RAD index of accessibility to radial highways 10) TAX full-value property-tax rate per $10,000 11) PTRATIO pupil-teacher ratio by town 12) B 1000(Bk - 0.63)^2 where Bk is the prop. of b. by town 13) LSTAT % lower status of the population Returns X, y : [n_samples, n_features], [n_class_labels] X is the feature matrix with 506 housing samples as rows and 13 feature columns. y is a 1-dimensional array of the continuous target variable MEDV Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/data/boston_housing_data/","title":"boston_housing_data: The Boston housing dataset for regression"},{"location":"user_guide/data/boston_housing_data/#boston_housing_data-the-boston-housing-dataset-for-regression","text":"A function that loads the boston_housing_data dataset into NumPy arrays. from mlxtend.data import boston_housing_data","title":"boston_housing_data: The Boston housing dataset for regression"},{"location":"user_guide/data/boston_housing_data/#overview","text":"The Boston Housing dataset for regression analysis. Features CRIM: per capita crime rate by town ZN: proportion of residential land zoned for lots over 25,000 sq.ft. INDUS: proportion of non-retail business acres per town CHAS: Charles River dummy variable (= 1 if tract bounds river; 0 otherwise) NOX: nitric oxides concentration (parts per 10 million) RM: average number of rooms per dwelling AGE: proportion of owner-occupied units built prior to 1940 DIS: weighted distances to five Boston employment centres RAD: index of accessibility to radial highways TAX: full-value property-tax rate per $10,000 PTRATIO: pupil-teacher ratio by town B: 1000(Bk - 0.63)^2 where Bk is the proportion of b. by town LSTAT: % lower status of the population Number of samples: 506 Target variable (continuous): MEDV, Median value of owner-occupied homes in $1000's","title":"Overview"},{"location":"user_guide/data/boston_housing_data/#references","text":"Source: https://archive.ics.uci.edu/ml/datasets/Wine Harrison, D. and Rubinfeld, D.L. 'Hedonic prices and the demand for clean air', J. Environ. Economics & Management, vol.5, 81-102, 1978.","title":"References"},{"location":"user_guide/data/boston_housing_data/#example-1-dataset-overview","text":"from mlxtend.data import boston_housing_data X, y = boston_housing_data() print('Dimensions: %s x %s' % (X.shape[0], X.shape[1])) print('1st row', X[0]) (506, 14) Dimensions: 506 x 13 1st row [ 6.32000000e-03 1.80000000e+01 2.31000000e+00 0.00000000e+00 5.38000000e-01 6.57500000e+00 6.52000000e+01 4.09000000e+00 1.00000000e+00 2.96000000e+02 1.53000000e+01 3.96900000e+02 4.98000000e+00]","title":"Example 1 - Dataset overview"},{"location":"user_guide/data/boston_housing_data/#api","text":"boston_housing_data() Boston Housing dataset. Source : https://archive.ics.uci.edu/ml/datasets/Housing Number of samples : 506 Continuous target variable : MEDV MEDV = Median value of owner-occupied homes in $1000's Dataset Attributes: 1) CRIM per capita crime rate by town 2) ZN proportion of residential land zoned for lots over 25,000 sq.ft. 3) INDUS proportion of non-retail business acres per town 4) CHAS Charles River dummy variable (= 1 if tract bounds river; 0 otherwise) 5) NOX nitric oxides concentration (parts per 10 million) 6) RM average number of rooms per dwelling 7) AGE proportion of owner-occupied units built prior to 1940 8) DIS weighted distances to five Boston employment centres 9) RAD index of accessibility to radial highways 10) TAX full-value property-tax rate per $10,000 11) PTRATIO pupil-teacher ratio by town 12) B 1000(Bk - 0.63)^2 where Bk is the prop. of b. by town 13) LSTAT % lower status of the population Returns X, y : [n_samples, n_features], [n_class_labels] X is the feature matrix with 506 housing samples as rows and 13 feature columns. y is a 1-dimensional array of the continuous target variable MEDV Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/data/boston_housing_data/","title":"API"},{"location":"user_guide/data/iris_data/","text":"iris_data: The 3-class iris dataset for classification A function that loads the iris dataset into NumPy arrays. from mlxtend.data import iris_data Overview The Iris dataset for classification. Features Sepal length Sepal width Petal length Petal width Number of samples: 150 Target variable (discrete): {50x Setosa, 50x Versicolor, 50x Virginica} References Source: https://archive.ics.uci.edu/ml/datasets/Iris Bache, K. & Lichman, M. (2013). UCI Machine Learning Repository. Irvine, CA: University of California, School of Information and Computer Science. Example 1 - Dataset overview from mlxtend.data import iris_data X, y = iris_data() print('Dimensions: %s x %s' % (X.shape[0], X.shape[1])) print('\\nHeader: %s' % ['sepal length', 'sepal width', 'petal length', 'petal width']) print('1st row', X[0]) Dimensions: 150 x 4 Header: ['sepal length', 'sepal width', 'petal length', 'petal width'] 1st row [5.1 3.5 1.4 0.2] import numpy as np print('Classes: Setosa, Versicolor, Virginica') print(np.unique(y)) print('Class distribution: %s' % np.bincount(y)) Classes: Setosa, Versicolor, Virginica [0 1 2] Class distribution: [50 50 50] API iris_data(version='uci') Iris flower dataset. Source : https://archive.ics.uci.edu/ml/datasets/Iris Number of samples : 150 Class labels : {0, 1, 2}, distribution: [50, 50, 50] 0 = setosa, 1 = versicolor, 2 = virginica. Dataset Attributes: 1) sepal length [cm] 2) sepal width [cm] 3) petal length [cm] 4) petal width [cm] Parameters version : string, optional (default: 'uci'). Version to use {'uci', 'corrected'}. 'uci' loads the dataset as deposited on the UCI machine learning repository, and 'corrected' provides the version that is consistent with Fisher's original paper. See Note for details. Returns X, y : [n_samples, n_features], [n_class_labels] X is the feature matrix with 150 flower samples as rows, and 4 feature columns sepal length, sepal width, petal length, and petal width. y is a 1-dimensional array of the class labels {0, 1, 2} Note The Iris dataset (originally collected by Edgar Anderson) and available in UCI's machine learning repository is different from the Iris dataset described in the original paper by R.A. Fisher [1]). Precisely, there are two data points (row number 34 and 37) in UCI's Machine Learning repository are different from the origianlly published Iris dataset. Also, the original version of the Iris Dataset, which can be loaded via version='corrected' is the same as the one in R. [1] . A. Fisher (1936). \"The use of multiple measurements in taxonomic problems\". Annals of Eugenics. 7 (2): 179\u2013188 Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/data/iris_data/","title":"iris_data: The 3-class iris dataset for classification"},{"location":"user_guide/data/iris_data/#iris_data-the-3-class-iris-dataset-for-classification","text":"A function that loads the iris dataset into NumPy arrays. from mlxtend.data import iris_data","title":"iris_data: The 3-class iris dataset for classification"},{"location":"user_guide/data/iris_data/#overview","text":"The Iris dataset for classification. Features Sepal length Sepal width Petal length Petal width Number of samples: 150 Target variable (discrete): {50x Setosa, 50x Versicolor, 50x Virginica}","title":"Overview"},{"location":"user_guide/data/iris_data/#references","text":"Source: https://archive.ics.uci.edu/ml/datasets/Iris Bache, K. & Lichman, M. (2013). UCI Machine Learning Repository. Irvine, CA: University of California, School of Information and Computer Science.","title":"References"},{"location":"user_guide/data/iris_data/#example-1-dataset-overview","text":"from mlxtend.data import iris_data X, y = iris_data() print('Dimensions: %s x %s' % (X.shape[0], X.shape[1])) print('\\nHeader: %s' % ['sepal length', 'sepal width', 'petal length', 'petal width']) print('1st row', X[0]) Dimensions: 150 x 4 Header: ['sepal length', 'sepal width', 'petal length', 'petal width'] 1st row [5.1 3.5 1.4 0.2] import numpy as np print('Classes: Setosa, Versicolor, Virginica') print(np.unique(y)) print('Class distribution: %s' % np.bincount(y)) Classes: Setosa, Versicolor, Virginica [0 1 2] Class distribution: [50 50 50]","title":"Example 1 - Dataset overview"},{"location":"user_guide/data/iris_data/#api","text":"iris_data(version='uci') Iris flower dataset. Source : https://archive.ics.uci.edu/ml/datasets/Iris Number of samples : 150 Class labels : {0, 1, 2}, distribution: [50, 50, 50] 0 = setosa, 1 = versicolor, 2 = virginica. Dataset Attributes: 1) sepal length [cm] 2) sepal width [cm] 3) petal length [cm] 4) petal width [cm] Parameters version : string, optional (default: 'uci'). Version to use {'uci', 'corrected'}. 'uci' loads the dataset as deposited on the UCI machine learning repository, and 'corrected' provides the version that is consistent with Fisher's original paper. See Note for details. Returns X, y : [n_samples, n_features], [n_class_labels] X is the feature matrix with 150 flower samples as rows, and 4 feature columns sepal length, sepal width, petal length, and petal width. y is a 1-dimensional array of the class labels {0, 1, 2} Note The Iris dataset (originally collected by Edgar Anderson) and available in UCI's machine learning repository is different from the Iris dataset described in the original paper by R.A. Fisher [1]). Precisely, there are two data points (row number 34 and 37) in UCI's Machine Learning repository are different from the origianlly published Iris dataset. Also, the original version of the Iris Dataset, which can be loaded via version='corrected' is the same as the one in R. [1] . A. Fisher (1936). \"The use of multiple measurements in taxonomic problems\". Annals of Eugenics. 7 (2): 179\u2013188 Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/data/iris_data/","title":"API"},{"location":"user_guide/data/loadlocal_mnist/","text":"loadlocal_mnist: A function for loading MNIST from the original ubyte files A utility function that loads the MNIST dataset from byte-form into NumPy arrays. from mlxtend.data import loadlocal_mnist Overview The MNIST dataset was constructed from two datasets of the US National Institute of Standards and Technology (NIST). The training set consists of handwritten digits from 250 different people, 50 percent high school students, and 50 percent employees from the Census Bureau. Note that the test set contains handwritten digits from different people following the same split. The MNIST dataset is publicly available at https://yann.lecun.com/exdb/mnist/ and consists of the following four parts: - Training set images: train-images-idx3-ubyte.gz (9.9 MB, 47 MB unzipped, and 60,000 samples) - Training set labels: train-labels-idx1-ubyte.gz (29 KB, 60 KB unzipped, and 60,000 labels) - Test set images: t10k-images-idx3-ubyte.gz (1.6 MB, 7.8 MB, unzipped and 10,000 samples) - Test set labels: t10k-labels-idx1-ubyte.gz (5 KB, 10 KB unzipped, and 10,000 labels) Features Each feature vector (row in the feature matrix) consists of 784 pixels (intensities) -- unrolled from the original 28x28 pixels images. Number of samples: 50000 images Target variable (discrete): Uniformly distributed class labels 0-9 corresponding to the respective handwritten digit shown in the image. References Source: https://yann.lecun.com/exdb/mnist/ Y. LeCun and C. Cortes. Mnist handwritten digit database. AT&T Labs [Online]. Available: https://yann. lecun. com/exdb/mnist, 2010. Example 1 Part 1 - Downloading the MNIST dataset 1) Download the MNIST files from Y. LeCun's website https://yann.lecun.com/exdb/mnist/train-images-idx3-ubyte.gz https://yann.lecun.com/exdb/mnist/train-labels-idx1-ubyte.gz https://yann.lecun.com/exdb/mnist/t10k-images-idx3-ubyte.gz https://yann.lecun.com/exdb/mnist/t10k-labels-idx1-ubyte.gz for example, via curl -O https://yann.lecun.com/exdb/mnist/train-images-idx3-ubyte.gz 2) Unzip the downloaded gzip archives for example, via gunzip t*-ubyte.gz Example 1 Part 2 - Loading MNIST into NumPy Arrays from mlxtend.data import loadlocal_mnist import platform if not platform.system() == 'Windows': X, y = loadlocal_mnist( images_path='train-images-idx3-ubyte', labels_path='train-labels-idx1-ubyte') else: X, y = loadlocal_mnist( images_path='train-images.idx3-ubyte', labels_path='train-labels.idx1-ubyte') print('Dimensions: %s x %s' % (X.shape[0], X.shape[1])) print('\\n1st row', X[0]) Dimensions: 60000 x 784 1st row [ 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 3 18 18 18 126 136 175 26 166 255 247 127 0 0 0 0 0 0 0 0 0 0 0 0 30 36 94 154 170 253 253 253 253 253 225 172 253 242 195 64 0 0 0 0 0 0 0 0 0 0 0 49 238 253 253 253 253 253 253 253 253 251 93 82 82 56 39 0 0 0 0 0 0 0 0 0 0 0 0 18 219 253 253 253 253 253 198 182 247 241 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 80 156 107 253 253 205 11 0 43 154 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 14 1 154 253 90 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 139 253 190 2 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 11 190 253 70 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 35 241 225 160 108 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 81 240 253 253 119 25 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 45 186 253 253 150 27 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 16 93 252 253 187 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 249 253 249 64 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 46 130 183 253 253 207 2 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 39 148 229 253 253 253 250 182 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 24 114 221 253 253 253 253 201 78 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 23 66 213 253 253 253 253 198 81 2 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 18 171 219 253 253 253 253 195 80 9 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 55 172 226 253 253 253 253 244 133 11 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 136 253 253 253 212 135 132 16 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0] import numpy as np print('Digits: 0 1 2 3 4 5 6 7 8 9') print('labels: %s' % np.unique(y)) print('Class distribution: %s' % np.bincount(y)) Digits: 0 1 2 3 4 5 6 7 8 9 labels: [0 1 2 3 4 5 6 7 8 9] Class distribution: [5923 6742 5958 6131 5842 5421 5918 6265 5851 5949] Store as CSV Files np.savetxt(fname='images.csv', X=X, delimiter=',', fmt='%d') np.savetxt(fname='labels.csv', X=y, delimiter=',', fmt='%d') API loadlocal_mnist(images_path, labels_path) Read MNIST from ubyte files. Parameters images_path : str path to the test or train MNIST ubyte file labels_path : str path to the test or train MNIST class labels file Returns images : [n_samples, n_pixels] numpy.array Pixel values of the images. labels : [n_samples] numpy array Target class labels Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/data/loadlocal_mnist/","title":"loadlocal_mnist: A function for loading MNIST from the original ubyte files"},{"location":"user_guide/data/loadlocal_mnist/#loadlocal_mnist-a-function-for-loading-mnist-from-the-original-ubyte-files","text":"A utility function that loads the MNIST dataset from byte-form into NumPy arrays. from mlxtend.data import loadlocal_mnist","title":"loadlocal_mnist: A function for loading MNIST from the original ubyte files"},{"location":"user_guide/data/loadlocal_mnist/#overview","text":"The MNIST dataset was constructed from two datasets of the US National Institute of Standards and Technology (NIST). The training set consists of handwritten digits from 250 different people, 50 percent high school students, and 50 percent employees from the Census Bureau. Note that the test set contains handwritten digits from different people following the same split. The MNIST dataset is publicly available at https://yann.lecun.com/exdb/mnist/ and consists of the following four parts: - Training set images: train-images-idx3-ubyte.gz (9.9 MB, 47 MB unzipped, and 60,000 samples) - Training set labels: train-labels-idx1-ubyte.gz (29 KB, 60 KB unzipped, and 60,000 labels) - Test set images: t10k-images-idx3-ubyte.gz (1.6 MB, 7.8 MB, unzipped and 10,000 samples) - Test set labels: t10k-labels-idx1-ubyte.gz (5 KB, 10 KB unzipped, and 10,000 labels) Features Each feature vector (row in the feature matrix) consists of 784 pixels (intensities) -- unrolled from the original 28x28 pixels images. Number of samples: 50000 images Target variable (discrete): Uniformly distributed class labels 0-9 corresponding to the respective handwritten digit shown in the image.","title":"Overview"},{"location":"user_guide/data/loadlocal_mnist/#references","text":"Source: https://yann.lecun.com/exdb/mnist/ Y. LeCun and C. Cortes. Mnist handwritten digit database. AT&T Labs [Online]. Available: https://yann. lecun. com/exdb/mnist, 2010.","title":"References"},{"location":"user_guide/data/loadlocal_mnist/#example-1-part-1-downloading-the-mnist-dataset","text":"1) Download the MNIST files from Y. LeCun's website https://yann.lecun.com/exdb/mnist/train-images-idx3-ubyte.gz https://yann.lecun.com/exdb/mnist/train-labels-idx1-ubyte.gz https://yann.lecun.com/exdb/mnist/t10k-images-idx3-ubyte.gz https://yann.lecun.com/exdb/mnist/t10k-labels-idx1-ubyte.gz for example, via curl -O https://yann.lecun.com/exdb/mnist/train-images-idx3-ubyte.gz 2) Unzip the downloaded gzip archives for example, via gunzip t*-ubyte.gz","title":"Example 1 Part 1 - Downloading the MNIST dataset"},{"location":"user_guide/data/loadlocal_mnist/#example-1-part-2-loading-mnist-into-numpy-arrays","text":"from mlxtend.data import loadlocal_mnist import platform if not platform.system() == 'Windows': X, y = loadlocal_mnist( images_path='train-images-idx3-ubyte', labels_path='train-labels-idx1-ubyte') else: X, y = loadlocal_mnist( images_path='train-images.idx3-ubyte', labels_path='train-labels.idx1-ubyte') print('Dimensions: %s x %s' % (X.shape[0], X.shape[1])) print('\\n1st row', X[0]) Dimensions: 60000 x 784 1st row [ 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 3 18 18 18 126 136 175 26 166 255 247 127 0 0 0 0 0 0 0 0 0 0 0 0 30 36 94 154 170 253 253 253 253 253 225 172 253 242 195 64 0 0 0 0 0 0 0 0 0 0 0 49 238 253 253 253 253 253 253 253 253 251 93 82 82 56 39 0 0 0 0 0 0 0 0 0 0 0 0 18 219 253 253 253 253 253 198 182 247 241 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 80 156 107 253 253 205 11 0 43 154 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 14 1 154 253 90 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 139 253 190 2 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 11 190 253 70 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 35 241 225 160 108 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 81 240 253 253 119 25 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 45 186 253 253 150 27 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 16 93 252 253 187 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 249 253 249 64 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 46 130 183 253 253 207 2 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 39 148 229 253 253 253 250 182 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 24 114 221 253 253 253 253 201 78 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 23 66 213 253 253 253 253 198 81 2 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 18 171 219 253 253 253 253 195 80 9 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 55 172 226 253 253 253 253 244 133 11 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 136 253 253 253 212 135 132 16 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0] import numpy as np print('Digits: 0 1 2 3 4 5 6 7 8 9') print('labels: %s' % np.unique(y)) print('Class distribution: %s' % np.bincount(y)) Digits: 0 1 2 3 4 5 6 7 8 9 labels: [0 1 2 3 4 5 6 7 8 9] Class distribution: [5923 6742 5958 6131 5842 5421 5918 6265 5851 5949]","title":"Example 1 Part 2 - Loading MNIST into NumPy Arrays"},{"location":"user_guide/data/loadlocal_mnist/#store-as-csv-files","text":"np.savetxt(fname='images.csv', X=X, delimiter=',', fmt='%d') np.savetxt(fname='labels.csv', X=y, delimiter=',', fmt='%d')","title":"Store as CSV Files"},{"location":"user_guide/data/loadlocal_mnist/#api","text":"loadlocal_mnist(images_path, labels_path) Read MNIST from ubyte files. Parameters images_path : str path to the test or train MNIST ubyte file labels_path : str path to the test or train MNIST class labels file Returns images : [n_samples, n_pixels] numpy.array Pixel values of the images. labels : [n_samples] numpy array Target class labels Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/data/loadlocal_mnist/","title":"API"},{"location":"user_guide/data/make_multiplexer_dataset/","text":"make_multiplexer_dataset: A function for creating multiplexer data Function that creates a dataset generated by a n-bit Boolean multiplexer for evaluating supervised learning algorithms. from mlxtend.data import make_multiplexer_dataset Overview The make_multiplexer_dataset function creates a dataset generated by an n-bit Boolean multiplexer. Such dataset represents a dataset generated by a simple rule, based on the behavior of a electric multiplexer, yet presents a relatively challenging classification problem for supervised learning algorithm with interactions between features (epistasis) as it may be encountered in many real-world scenarios [1]. The following illustration depicts a 6-bit multiplexer that consists of 2 address bits and 4 register bits. The address bits converted to decimal representation point to a position in the register bit. For example, if the address bits are \"00\" (0 in decimal), the address bits point to the register bit at position 0. The value of the register position pointed to determines the class label. For example, if the register bit at position is 0, the class label is 0. Vice versa, if the register bit at position 0 is 1, the class label is 1. In the example above, the address bits \"10\" (2 in decimal) point to the 3rd register position (as we start counting from index 0), which has a bit value of 1. Hence, the class label is 1. Below are a few more examples: Address bits: [0, 1], register bits: [1, 0, 1, 1], class label: 0 Address bits: [0, 1], register bits: [1, 1, 1, 0], class label: 1 Address bits: [1, 0], register bits: [1, 0, 0, 1], class label: 0 Address bits: [1, 1], register bits: [1, 1, 1, 0], class label: 0 Address bits: [0, 1], register bits: [0, 1, 1, 0], class label: 1 Address bits: [0, 1], register bits: [1, 0, 0, 1], class label: 0 Address bits: [0, 1], register bits: [0, 1, 1, 1], class label: 1 Address bits: [0, 1], register bits: [0, 0, 0, 0], class label: 0 Address bits: [1, 0], register bits: [1, 0, 1, 1], class label: 1 Address bits: [0, 1], register bits: [1, 1, 1, 1], class label: 1 Note that in the implementation of the multiplexer function, if the number of address bits is set to 2, this results in a 6 bit multiplexer as two bit can have 2^2=4 different register positions (2 bit + 4 bit = 6 bit). However, if we choose 3 address bits instead, 2^3=8 positions would be covered, resulting in a 11 bit (3 bit + 8 bit = 11 bit) multiplexer, and so forth. References [1] Urbanowicz, R. J., & Browne, W. N. (2017). Introduction to Learning Classifier Systems . Springer. Example 1 -- 6-bit multiplexer This simple example illustrates how to create dataset from a 6-bit multiplexer import numpy as np from mlxtend.data import make_multiplexer_dataset X, y = make_multiplexer_dataset(address_bits=2, sample_size=10, positive_class_ratio=0.5, shuffle=False, random_seed=123) print('Features:\\n', X) print('\\nClass labels:\\n', y) Features: [[0 1 0 1 0 1] [1 0 0 0 1 1] [0 1 1 1 0 0] [0 1 1 1 0 0] [0 0 1 1 0 0] [0 1 0 0 0 0] [0 1 1 0 1 1] [1 0 1 0 0 0] [1 0 0 1 0 1] [1 0 1 0 0 1]] Class labels: [1 1 1 1 1 0 0 0 0 0] API make_multiplexer_dataset(address_bits=2, sample_size=100, positive_class_ratio=0.5, shuffle=False, random_seed=None) Function to create a binary n-bit multiplexer dataset. New in mlxtend v0.9 Parameters address_bits : int (default: 2) A positive integer that determines the number of address bits in the multiplexer, which in turn determine the n-bit capacity of the multiplexer and therefore the number of features. The number of features is determined by the number of address bits. For example, 2 address bits will result in a 6 bit multiplexer and consequently 6 features (2 + 2^2 = 6). If address_bits=3 , then this results in an 11-bit multiplexer as (2 + 2^3 = 11) with 11 features. sample_size : int (default: 100) The total number of samples generated. positive_class_ratio : float (default: 0.5) The fraction (a float between 0 and 1) of samples in the sample_size d dataset that have class label 1. If positive_class_ratio=0.5 (default), then the ratio of class 0 and class 1 samples is perfectly balanced. shuffle : Bool (default: False) Whether or not to shuffle the features and labels. If False (default), the samples are returned in sorted order starting with sample_size /2 samples with class label 0 and followed by sample_size /2 samples with class label 1. random_seed : int (default: None) Random seed used for generating the multiplexer samples and shuffling. Returns X, y : [n_samples, n_features], [n_class_labels] X is the feature matrix with the number of samples equal to sample_size . The number of features is determined by the number of address bits. For instance, 2 address bits will result in a 6 bit multiplexer and consequently 6 features (2 + 2^2 = 6). All features are binary (values in {0, 1}). y is a 1-dimensional array of class labels in {0, 1}. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/data/make_multiplexer_dataset","title":"make_multiplexer_dataset: A function for creating multiplexer data"},{"location":"user_guide/data/make_multiplexer_dataset/#make_multiplexer_dataset-a-function-for-creating-multiplexer-data","text":"Function that creates a dataset generated by a n-bit Boolean multiplexer for evaluating supervised learning algorithms. from mlxtend.data import make_multiplexer_dataset","title":"make_multiplexer_dataset: A function for creating multiplexer data"},{"location":"user_guide/data/make_multiplexer_dataset/#overview","text":"The make_multiplexer_dataset function creates a dataset generated by an n-bit Boolean multiplexer. Such dataset represents a dataset generated by a simple rule, based on the behavior of a electric multiplexer, yet presents a relatively challenging classification problem for supervised learning algorithm with interactions between features (epistasis) as it may be encountered in many real-world scenarios [1]. The following illustration depicts a 6-bit multiplexer that consists of 2 address bits and 4 register bits. The address bits converted to decimal representation point to a position in the register bit. For example, if the address bits are \"00\" (0 in decimal), the address bits point to the register bit at position 0. The value of the register position pointed to determines the class label. For example, if the register bit at position is 0, the class label is 0. Vice versa, if the register bit at position 0 is 1, the class label is 1. In the example above, the address bits \"10\" (2 in decimal) point to the 3rd register position (as we start counting from index 0), which has a bit value of 1. Hence, the class label is 1. Below are a few more examples: Address bits: [0, 1], register bits: [1, 0, 1, 1], class label: 0 Address bits: [0, 1], register bits: [1, 1, 1, 0], class label: 1 Address bits: [1, 0], register bits: [1, 0, 0, 1], class label: 0 Address bits: [1, 1], register bits: [1, 1, 1, 0], class label: 0 Address bits: [0, 1], register bits: [0, 1, 1, 0], class label: 1 Address bits: [0, 1], register bits: [1, 0, 0, 1], class label: 0 Address bits: [0, 1], register bits: [0, 1, 1, 1], class label: 1 Address bits: [0, 1], register bits: [0, 0, 0, 0], class label: 0 Address bits: [1, 0], register bits: [1, 0, 1, 1], class label: 1 Address bits: [0, 1], register bits: [1, 1, 1, 1], class label: 1 Note that in the implementation of the multiplexer function, if the number of address bits is set to 2, this results in a 6 bit multiplexer as two bit can have 2^2=4 different register positions (2 bit + 4 bit = 6 bit). However, if we choose 3 address bits instead, 2^3=8 positions would be covered, resulting in a 11 bit (3 bit + 8 bit = 11 bit) multiplexer, and so forth.","title":"Overview"},{"location":"user_guide/data/make_multiplexer_dataset/#references","text":"[1] Urbanowicz, R. J., & Browne, W. N. (2017). Introduction to Learning Classifier Systems . Springer.","title":"References"},{"location":"user_guide/data/make_multiplexer_dataset/#example-1-6-bit-multiplexer","text":"This simple example illustrates how to create dataset from a 6-bit multiplexer import numpy as np from mlxtend.data import make_multiplexer_dataset X, y = make_multiplexer_dataset(address_bits=2, sample_size=10, positive_class_ratio=0.5, shuffle=False, random_seed=123) print('Features:\\n', X) print('\\nClass labels:\\n', y) Features: [[0 1 0 1 0 1] [1 0 0 0 1 1] [0 1 1 1 0 0] [0 1 1 1 0 0] [0 0 1 1 0 0] [0 1 0 0 0 0] [0 1 1 0 1 1] [1 0 1 0 0 0] [1 0 0 1 0 1] [1 0 1 0 0 1]] Class labels: [1 1 1 1 1 0 0 0 0 0]","title":"Example 1 -- 6-bit multiplexer"},{"location":"user_guide/data/make_multiplexer_dataset/#api","text":"make_multiplexer_dataset(address_bits=2, sample_size=100, positive_class_ratio=0.5, shuffle=False, random_seed=None) Function to create a binary n-bit multiplexer dataset. New in mlxtend v0.9 Parameters address_bits : int (default: 2) A positive integer that determines the number of address bits in the multiplexer, which in turn determine the n-bit capacity of the multiplexer and therefore the number of features. The number of features is determined by the number of address bits. For example, 2 address bits will result in a 6 bit multiplexer and consequently 6 features (2 + 2^2 = 6). If address_bits=3 , then this results in an 11-bit multiplexer as (2 + 2^3 = 11) with 11 features. sample_size : int (default: 100) The total number of samples generated. positive_class_ratio : float (default: 0.5) The fraction (a float between 0 and 1) of samples in the sample_size d dataset that have class label 1. If positive_class_ratio=0.5 (default), then the ratio of class 0 and class 1 samples is perfectly balanced. shuffle : Bool (default: False) Whether or not to shuffle the features and labels. If False (default), the samples are returned in sorted order starting with sample_size /2 samples with class label 0 and followed by sample_size /2 samples with class label 1. random_seed : int (default: None) Random seed used for generating the multiplexer samples and shuffling. Returns X, y : [n_samples, n_features], [n_class_labels] X is the feature matrix with the number of samples equal to sample_size . The number of features is determined by the number of address bits. For instance, 2 address bits will result in a 6 bit multiplexer and consequently 6 features (2 + 2^2 = 6). All features are binary (values in {0, 1}). y is a 1-dimensional array of class labels in {0, 1}. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/data/make_multiplexer_dataset","title":"API"},{"location":"user_guide/data/mnist_data/","text":"mnist_data: A subset of the MNIST dataset for classification A function that loads the MNIST dataset into NumPy arrays. from mlxtend.data import mnist_data Overview The MNIST dataset was constructed from two datasets of the US National Institute of Standards and Technology (NIST). The training set consists of handwritten digits from 250 different people, 50 percent high school students, and 50 percent employees from the Census Bureau. Note that the test set contains handwritten digits from different people following the same split. Features Each feature vector (row in the feature matrix) consists of 784 pixels (intensities) -- unrolled from the original 28x28 pixels images. Number of samples: A subset of 5000 images (the first 500 digits of each class) Target variable (discrete): {500x 0, ..., 500x 9} References Source: https://yann.lecun.com/exdb/mnist/ Y. LeCun and C. Cortes. Mnist handwritten digit database. AT&T Labs [Online]. Available: https://yann.lecun.com/exdb/mnist , 2010. Example 1 - Dataset overview from mlxtend.data import mnist_data X, y = mnist_data() print('Dimensions: %s x %s' % (X.shape[0], X.shape[1])) print('1st row', X[0]) Dimensions: 5000 x 784 1st row [ 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 51. 159. 253. 159. 50. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 48. 238. 252. 252. 252. 237. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 54. 227. 253. 252. 239. 233. 252. 57. 6. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 10. 60. 224. 252. 253. 252. 202. 84. 252. 253. 122. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 163. 252. 252. 252. 253. 252. 252. 96. 189. 253. 167. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 51. 238. 253. 253. 190. 114. 253. 228. 47. 79. 255. 168. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 48. 238. 252. 252. 179. 12. 75. 121. 21. 0. 0. 253. 243. 50. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 38. 165. 253. 233. 208. 84. 0. 0. 0. 0. 0. 0. 253. 252. 165. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 7. 178. 252. 240. 71. 19. 28. 0. 0. 0. 0. 0. 0. 253. 252. 195. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 57. 252. 252. 63. 0. 0. 0. 0. 0. 0. 0. 0. 0. 253. 252. 195. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 198. 253. 190. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 255. 253. 196. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 76. 246. 252. 112. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 253. 252. 148. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 85. 252. 230. 25. 0. 0. 0. 0. 0. 0. 0. 0. 7. 135. 253. 186. 12. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 85. 252. 223. 0. 0. 0. 0. 0. 0. 0. 0. 7. 131. 252. 225. 71. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 85. 252. 145. 0. 0. 0. 0. 0. 0. 0. 48. 165. 252. 173. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 86. 253. 225. 0. 0. 0. 0. 0. 0. 114. 238. 253. 162. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 85. 252. 249. 146. 48. 29. 85. 178. 225. 253. 223. 167. 56. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 85. 252. 252. 252. 229. 215. 252. 252. 252. 196. 130. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 28. 199. 252. 252. 253. 252. 252. 233. 145. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 25. 128. 252. 253. 252. 141. 37. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.] import numpy as np print('Classes: Setosa, Versicolor, Virginica') print(np.unique(y)) print('Class distribution: %s' % np.bincount(y)) Classes: Setosa, Versicolor, Virginica [0 1 2 3 4 5 6 7 8 9] Class distribution: [500 500 500 500 500 500 500 500 500 500] Example 2 - Visualize MNIST %matplotlib inline import matplotlib.pyplot as plt def plot_digit(X, y, idx): img = X[idx].reshape(28,28) plt.imshow(img, cmap='Greys', interpolation='nearest') plt.title('true label: %d' % y[idx]) plt.show() plot_digit(X, y, 4) API mnist_data() 5000 samples from the MNIST handwritten digits dataset. Data Source : https://yann.lecun.com/exdb/mnist/ Returns X, y : [n_samples, n_features], [n_class_labels] X is the feature matrix with 5000 image samples as rows, each row consists of 28x28 pixels that were unrolled into 784 pixel feature vectors. y contains the 10 unique class labels 0-9. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/data/mnist_data/","title":"mnist_data: A subset of the MNIST dataset for classification"},{"location":"user_guide/data/mnist_data/#mnist_data-a-subset-of-the-mnist-dataset-for-classification","text":"A function that loads the MNIST dataset into NumPy arrays. from mlxtend.data import mnist_data","title":"mnist_data: A subset of the MNIST dataset for classification"},{"location":"user_guide/data/mnist_data/#overview","text":"The MNIST dataset was constructed from two datasets of the US National Institute of Standards and Technology (NIST). The training set consists of handwritten digits from 250 different people, 50 percent high school students, and 50 percent employees from the Census Bureau. Note that the test set contains handwritten digits from different people following the same split. Features Each feature vector (row in the feature matrix) consists of 784 pixels (intensities) -- unrolled from the original 28x28 pixels images. Number of samples: A subset of 5000 images (the first 500 digits of each class) Target variable (discrete): {500x 0, ..., 500x 9}","title":"Overview"},{"location":"user_guide/data/mnist_data/#references","text":"Source: https://yann.lecun.com/exdb/mnist/ Y. LeCun and C. Cortes. Mnist handwritten digit database. AT&T Labs [Online]. Available: https://yann.lecun.com/exdb/mnist , 2010.","title":"References"},{"location":"user_guide/data/mnist_data/#example-1-dataset-overview","text":"from mlxtend.data import mnist_data X, y = mnist_data() print('Dimensions: %s x %s' % (X.shape[0], X.shape[1])) print('1st row', X[0]) Dimensions: 5000 x 784 1st row [ 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 51. 159. 253. 159. 50. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 48. 238. 252. 252. 252. 237. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 54. 227. 253. 252. 239. 233. 252. 57. 6. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 10. 60. 224. 252. 253. 252. 202. 84. 252. 253. 122. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 163. 252. 252. 252. 253. 252. 252. 96. 189. 253. 167. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 51. 238. 253. 253. 190. 114. 253. 228. 47. 79. 255. 168. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 48. 238. 252. 252. 179. 12. 75. 121. 21. 0. 0. 253. 243. 50. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 38. 165. 253. 233. 208. 84. 0. 0. 0. 0. 0. 0. 253. 252. 165. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 7. 178. 252. 240. 71. 19. 28. 0. 0. 0. 0. 0. 0. 253. 252. 195. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 57. 252. 252. 63. 0. 0. 0. 0. 0. 0. 0. 0. 0. 253. 252. 195. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 198. 253. 190. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 255. 253. 196. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 76. 246. 252. 112. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 253. 252. 148. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 85. 252. 230. 25. 0. 0. 0. 0. 0. 0. 0. 0. 7. 135. 253. 186. 12. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 85. 252. 223. 0. 0. 0. 0. 0. 0. 0. 0. 7. 131. 252. 225. 71. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 85. 252. 145. 0. 0. 0. 0. 0. 0. 0. 48. 165. 252. 173. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 86. 253. 225. 0. 0. 0. 0. 0. 0. 114. 238. 253. 162. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 85. 252. 249. 146. 48. 29. 85. 178. 225. 253. 223. 167. 56. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 85. 252. 252. 252. 229. 215. 252. 252. 252. 196. 130. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 28. 199. 252. 252. 253. 252. 252. 233. 145. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 25. 128. 252. 253. 252. 141. 37. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.] import numpy as np print('Classes: Setosa, Versicolor, Virginica') print(np.unique(y)) print('Class distribution: %s' % np.bincount(y)) Classes: Setosa, Versicolor, Virginica [0 1 2 3 4 5 6 7 8 9] Class distribution: [500 500 500 500 500 500 500 500 500 500]","title":"Example 1 - Dataset overview"},{"location":"user_guide/data/mnist_data/#example-2-visualize-mnist","text":"%matplotlib inline import matplotlib.pyplot as plt def plot_digit(X, y, idx): img = X[idx].reshape(28,28) plt.imshow(img, cmap='Greys', interpolation='nearest') plt.title('true label: %d' % y[idx]) plt.show() plot_digit(X, y, 4)","title":"Example 2 - Visualize MNIST"},{"location":"user_guide/data/mnist_data/#api","text":"mnist_data() 5000 samples from the MNIST handwritten digits dataset. Data Source : https://yann.lecun.com/exdb/mnist/ Returns X, y : [n_samples, n_features], [n_class_labels] X is the feature matrix with 5000 image samples as rows, each row consists of 28x28 pixels that were unrolled into 784 pixel feature vectors. y contains the 10 unique class labels 0-9. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/data/mnist_data/","title":"API"},{"location":"user_guide/data/three_blobs_data/","text":"three_blobs_data: The synthetic blobs for classification A function that loads the three_blobs dataset into NumPy arrays. from mlxtend.data import three_blobs_data Overview A random dataset of 3 2D blobs for clustering. Number of samples : 150 Suggested labels \\in {0, 1, 2}, distribution: [50, 50, 50] References Example 1 - Dataset overview from mlxtend.data import three_blobs_data X, y = three_blobs_data() print('Dimensions: %s x %s' % (X.shape[0], X.shape[1])) print('1st row', X[0]) Dimensions: 150 x 2 1st row [ 2.60509732 1.22529553] import numpy as np print('Suggested cluster labels') print(np.unique(y)) print('Label distribution: %s' % np.bincount(y)) Suggested cluster labels [0 1 2] Label distribution: [50 50 50] import matplotlib.pyplot as plt plt.scatter(X[:,0], X[:,1], c='white', marker='o', s=50) plt.grid() plt.show() plt.scatter(X[y == 0, 0], X[y == 0, 1], s=50, c='lightgreen', marker='s', label='cluster 1') plt.scatter(X[y == 1,0], X[y == 1,1], s=50, c='orange', marker='o', label='cluster 2') plt.scatter(X[y == 2,0], X[y == 2,1], s=50, c='lightblue', marker='v', label='cluster 3') plt.legend(loc='lower left') plt.grid() plt.show() API three_blobs_data() A random dataset of 3 2D blobs for clustering. Number of samples : 150 Suggested labels : {0, 1, 2}, distribution: [50, 50, 50] Returns X, y : [n_samples, n_features], [n_cluster_labels] X is the feature matrix with 159 samples as rows and 2 feature columns. y is a 1-dimensional array of the 3 suggested cluster labels 0, 1, 2 Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/data/three_blobs_data","title":"three_blobs_data: The synthetic blobs for classification"},{"location":"user_guide/data/three_blobs_data/#three_blobs_data-the-synthetic-blobs-for-classification","text":"A function that loads the three_blobs dataset into NumPy arrays. from mlxtend.data import three_blobs_data","title":"three_blobs_data: The synthetic blobs for classification"},{"location":"user_guide/data/three_blobs_data/#overview","text":"A random dataset of 3 2D blobs for clustering. Number of samples : 150 Suggested labels \\in {0, 1, 2}, distribution: [50, 50, 50]","title":"Overview"},{"location":"user_guide/data/three_blobs_data/#references","text":"","title":"References"},{"location":"user_guide/data/three_blobs_data/#example-1-dataset-overview","text":"from mlxtend.data import three_blobs_data X, y = three_blobs_data() print('Dimensions: %s x %s' % (X.shape[0], X.shape[1])) print('1st row', X[0]) Dimensions: 150 x 2 1st row [ 2.60509732 1.22529553] import numpy as np print('Suggested cluster labels') print(np.unique(y)) print('Label distribution: %s' % np.bincount(y)) Suggested cluster labels [0 1 2] Label distribution: [50 50 50] import matplotlib.pyplot as plt plt.scatter(X[:,0], X[:,1], c='white', marker='o', s=50) plt.grid() plt.show() plt.scatter(X[y == 0, 0], X[y == 0, 1], s=50, c='lightgreen', marker='s', label='cluster 1') plt.scatter(X[y == 1,0], X[y == 1,1], s=50, c='orange', marker='o', label='cluster 2') plt.scatter(X[y == 2,0], X[y == 2,1], s=50, c='lightblue', marker='v', label='cluster 3') plt.legend(loc='lower left') plt.grid() plt.show()","title":"Example 1 - Dataset overview"},{"location":"user_guide/data/three_blobs_data/#api","text":"three_blobs_data() A random dataset of 3 2D blobs for clustering. Number of samples : 150 Suggested labels : {0, 1, 2}, distribution: [50, 50, 50] Returns X, y : [n_samples, n_features], [n_cluster_labels] X is the feature matrix with 159 samples as rows and 2 feature columns. y is a 1-dimensional array of the 3 suggested cluster labels 0, 1, 2 Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/data/three_blobs_data","title":"API"},{"location":"user_guide/data/wine_data/","text":"wine_data: A 3-class wine dataset for classification A function that loads the Wine dataset into NumPy arrays. from mlxtend.data import wine_data Overview The Wine dataset for classification. Samples 178 Features 13 Classes 3 Data Set Characteristics: Multivariate Attribute Characteristics: Integer, Real Associated Tasks: Classification Missing Values None column attribute 1) Class Label 2) Alcohol 3) Malic acid 4) Ash 5) Alcalinity of ash 6) Magnesium 7) Total phenols 8) Flavanoids 9) Nonflavanoid phenols 10) Proanthocyanins 11) Color intensity 12) Hue 13) OD280/OD315 of diluted wines 14) Proline class samples 0 59 1 71 2 48 References Forina, M. et al, PARVUS - An Extendible Package for Data Exploration, Classification and Correlation. Institute of Pharmaceutical and Food Analysis and Technologies, Via Brigata Salerno, 16147 Genoa, Italy. Source: https://archive.ics.uci.edu/ml/datasets/Wine Bache, K. & Lichman, M. (2013). UCI Machine Learning Repository. Irvine, CA: University of California, School of Information and Computer Science. Example 1 - Dataset overview from mlxtend.data import wine_data X, y = wine_data() print('Dimensions: %s x %s' % (X.shape[0], X.shape[1])) print('\\nHeader: %s' % ['alcohol', 'malic acid', 'ash', 'ash alcalinity', 'magnesium', 'total phenols', 'flavanoids', 'nonflavanoid phenols', 'proanthocyanins', 'color intensity', 'hue', 'OD280/OD315 of diluted wines', 'proline']) print('1st row', X[0]) Dimensions: 178 x 13 Header: ['alcohol', 'malic acid', 'ash', 'ash alcalinity', 'magnesium', 'total phenols', 'flavanoids', 'nonflavanoid phenols', 'proanthocyanins', 'color intensity', 'hue', 'OD280/OD315 of diluted wines', 'proline'] 1st row [ 1.42300000e+01 1.71000000e+00 2.43000000e+00 1.56000000e+01 1.27000000e+02 2.80000000e+00 3.06000000e+00 2.80000000e-01 2.29000000e+00 5.64000000e+00 1.04000000e+00 3.92000000e+00 1.06500000e+03] import numpy as np print('Classes: %s' % np.unique(y)) print('Class distribution: %s' % np.bincount(y)) Classes: [0 1 2] Class distribution: [59 71 48] API wine_data() Wine dataset. Source : https://archive.ics.uci.edu/ml/datasets/Wine Number of samples : 178 Class labels : {0, 1, 2}, distribution: [59, 71, 48] Dataset Attributes: 1) Alcohol 2) Malic acid 3) Ash 4) Alcalinity of ash 5) Magnesium 6) Total phenols 7) Flavanoids 8) Nonflavanoid phenols 9) Proanthocyanins 10) Color intensity 11) Hue 12) OD280/OD315 of diluted wines 13) Proline Returns X, y : [n_samples, n_features], [n_class_labels] X is the feature matrix with 178 wine samples as rows and 13 feature columns. y is a 1-dimensional array of the 3 class labels 0, 1, 2 Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/data/wine_data","title":"wine_data: A 3-class wine dataset for classification"},{"location":"user_guide/data/wine_data/#wine_data-a-3-class-wine-dataset-for-classification","text":"A function that loads the Wine dataset into NumPy arrays. from mlxtend.data import wine_data","title":"wine_data: A 3-class wine dataset for classification"},{"location":"user_guide/data/wine_data/#overview","text":"The Wine dataset for classification. Samples 178 Features 13 Classes 3 Data Set Characteristics: Multivariate Attribute Characteristics: Integer, Real Associated Tasks: Classification Missing Values None column attribute 1) Class Label 2) Alcohol 3) Malic acid 4) Ash 5) Alcalinity of ash 6) Magnesium 7) Total phenols 8) Flavanoids 9) Nonflavanoid phenols 10) Proanthocyanins 11) Color intensity 12) Hue 13) OD280/OD315 of diluted wines 14) Proline class samples 0 59 1 71 2 48","title":"Overview"},{"location":"user_guide/data/wine_data/#references","text":"Forina, M. et al, PARVUS - An Extendible Package for Data Exploration, Classification and Correlation. Institute of Pharmaceutical and Food Analysis and Technologies, Via Brigata Salerno, 16147 Genoa, Italy. Source: https://archive.ics.uci.edu/ml/datasets/Wine Bache, K. & Lichman, M. (2013). UCI Machine Learning Repository. Irvine, CA: University of California, School of Information and Computer Science.","title":"References"},{"location":"user_guide/data/wine_data/#example-1-dataset-overview","text":"from mlxtend.data import wine_data X, y = wine_data() print('Dimensions: %s x %s' % (X.shape[0], X.shape[1])) print('\\nHeader: %s' % ['alcohol', 'malic acid', 'ash', 'ash alcalinity', 'magnesium', 'total phenols', 'flavanoids', 'nonflavanoid phenols', 'proanthocyanins', 'color intensity', 'hue', 'OD280/OD315 of diluted wines', 'proline']) print('1st row', X[0]) Dimensions: 178 x 13 Header: ['alcohol', 'malic acid', 'ash', 'ash alcalinity', 'magnesium', 'total phenols', 'flavanoids', 'nonflavanoid phenols', 'proanthocyanins', 'color intensity', 'hue', 'OD280/OD315 of diluted wines', 'proline'] 1st row [ 1.42300000e+01 1.71000000e+00 2.43000000e+00 1.56000000e+01 1.27000000e+02 2.80000000e+00 3.06000000e+00 2.80000000e-01 2.29000000e+00 5.64000000e+00 1.04000000e+00 3.92000000e+00 1.06500000e+03] import numpy as np print('Classes: %s' % np.unique(y)) print('Class distribution: %s' % np.bincount(y)) Classes: [0 1 2] Class distribution: [59 71 48]","title":"Example 1 - Dataset overview"},{"location":"user_guide/data/wine_data/#api","text":"wine_data() Wine dataset. Source : https://archive.ics.uci.edu/ml/datasets/Wine Number of samples : 178 Class labels : {0, 1, 2}, distribution: [59, 71, 48] Dataset Attributes: 1) Alcohol 2) Malic acid 3) Ash 4) Alcalinity of ash 5) Magnesium 6) Total phenols 7) Flavanoids 8) Nonflavanoid phenols 9) Proanthocyanins 10) Color intensity 11) Hue 12) OD280/OD315 of diluted wines 13) Proline Returns X, y : [n_samples, n_features], [n_class_labels] X is the feature matrix with 178 wine samples as rows and 13 feature columns. y is a 1-dimensional array of the 3 class labels 0, 1, 2 Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/data/wine_data","title":"API"},{"location":"user_guide/evaluate/BootstrapOutOfBag/","text":"BootstrapOutOfBag: A scikit-learn compatible version of the out-of-bag bootstrap An implementation of the out-of-bag bootstrap to evaluate supervised learning algorithms. from mlxtend.evaluate import BootstrapOutOfBag Overview Originally, the bootstrap method aims to determine the statistical properties of an estimator when the underlying distribution was unknown and additional samples are not available. Now, in order to exploit this method for the evaluation of predictive models, such as hypotheses for classification and regression, we may prefer a slightly different approach to bootstrapping using the so-called Out-Of-Bag (OOB) or Leave-One-Out Bootstrap (LOOB) technique. Here, we use out-of-bag samples as test sets for evaluation instead of evaluating the model on the training data. Out-of-bag samples are the unique sets of instances that are not used for model fitting as shown in the figure below [1]. The figure above illustrates how three random bootstrap samples drawn from an exemplary ten-sample dataset ( X_1,X_2, ..., X_{10} ) and their out-of-bag sample for testing may look like. In practice, Bradley Efron and Robert Tibshirani recommend drawing 50 to 200 bootstrap samples as being sufficient for reliable estimates [2]. References [1] https://sebastianraschka.com/blog/2016/model-evaluation-selection-part2.html [2] Efron, Bradley, and Robert J. Tibshirani. An introduction to the bootstrap. CRC press, 1994. Management of Data (ACM SIGMOD '97), pages 265-276, 1997. Example 1 -- Evaluating the predictive performance of a model The BootstrapOutOfBag class mimics the behavior of scikit-learn's cross-validation classes, e.g., KFold : from mlxtend.evaluate import BootstrapOutOfBag import numpy as np oob = BootstrapOutOfBag(n_splits=3) for train, test in oob.split(np.array([1, 2, 3, 4, 5])): print(train, test) [4 2 1 3 3] [0] [2 4 1 2 1] [0 3] [4 3 3 4 1] [0 2] Consequently, we can use BootstrapOutOfBag objects via the cross_val_score method: from sklearn.datasets import load_iris from sklearn.linear_model import LogisticRegression from sklearn.model_selection import cross_val_score iris = load_iris() X = iris.data y = iris.target lr = LogisticRegression() print(cross_val_score(lr, X, y)) [ 0.96078431 0.92156863 0.95833333] print(cross_val_score(lr, X, y, cv=BootstrapOutOfBag(n_splits=3, random_seed=456))) [ 0.92727273 0.96226415 0.94444444] In practice, it is recommended to run at least 200 iterations, though: print('Mean accuracy: %.1f%%' % np.mean(100*cross_val_score( lr, X, y, cv=BootstrapOutOfBag(n_splits=200, random_seed=456)))) Mean accuracy: 94.8% Using the bootstrap, we can use the percentile method to compute the confidence bounds of the performance estimate. We pick our lower and upper confidence bounds as follows: ACC_{lower} = \\alpha_1th percentile of the ACC_{boot} distribution ACC_{lower} = \\alpha_2th percentile of the ACC_{boot} distribution where \\alpha_1 = \\alpha and \\alpha_2 = 1-\\alpha , and the degree of confidence to compute the 100 \\times (1-2 \\times \\alpha) confidence interval. For instance, to compute a 95% confidence interval, we pick \\alpha=0.025 to obtain the 2.5th and 97.5th percentiles of the b bootstrap samples distribution as the upper and lower confidence bounds. import matplotlib.pyplot as plt %matplotlib inline accuracies = cross_val_score(lr, X, y, cv=BootstrapOutOfBag(n_splits=1000, random_seed=456)) mean = np.mean(accuracies) lower = np.percentile(accuracies, 2.5) upper = np.percentile(accuracies, 97.5) fig, ax = plt.subplots(figsize=(8, 4)) ax.vlines(mean, [0], 40, lw=2.5, linestyle='-', label='mean') ax.vlines(lower, [0], 15, lw=2.5, linestyle='-.', label='CI95 percentile') ax.vlines(upper, [0], 15, lw=2.5, linestyle='-.') ax.hist(accuracies, bins=11, color='#0080ff', edgecolor=\"none\", alpha=0.3) plt.legend(loc='upper left') plt.show() API BootstrapOutOfBag(n_splits=200, random_seed=None) Parameters n_splits : int (default=200) Number of bootstrap iterations. Must be larger than 1. random_seed : int (default=None) If int, random_seed is the seed used by the random number generator. Returns train_idx : ndarray The training set indices for that split. test_idx : ndarray The testing set indices for that split. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/BootstrapOutOfBag/ Methods get_n_splits(X=None, y=None, groups=None) Returns the number of splitting iterations in the cross-validator Parameters X : object Always ignored, exists for compatibility with scikit-learn. y : object Always ignored, exists for compatibility with scikit-learn. groups : object Always ignored, exists for compatibility with scikit-learn. Returns n_splits : int Returns the number of splitting iterations in the cross-validator. split(X, y=None, groups=None) y : array-like or None (default: None) Argument is not used and only included as parameter for compatibility, similar to KFold in scikit-learn. groups : array-like or None (default: None) Argument is not used and only included as parameter for compatibility, similar to KFold in scikit-learn.","title":"BootstrapOutOfBag: A scikit-learn compatible version of the out-of-bag bootstrap"},{"location":"user_guide/evaluate/BootstrapOutOfBag/#bootstrapoutofbag-a-scikit-learn-compatible-version-of-the-out-of-bag-bootstrap","text":"An implementation of the out-of-bag bootstrap to evaluate supervised learning algorithms. from mlxtend.evaluate import BootstrapOutOfBag","title":"BootstrapOutOfBag: A scikit-learn compatible version of the out-of-bag bootstrap"},{"location":"user_guide/evaluate/BootstrapOutOfBag/#overview","text":"Originally, the bootstrap method aims to determine the statistical properties of an estimator when the underlying distribution was unknown and additional samples are not available. Now, in order to exploit this method for the evaluation of predictive models, such as hypotheses for classification and regression, we may prefer a slightly different approach to bootstrapping using the so-called Out-Of-Bag (OOB) or Leave-One-Out Bootstrap (LOOB) technique. Here, we use out-of-bag samples as test sets for evaluation instead of evaluating the model on the training data. Out-of-bag samples are the unique sets of instances that are not used for model fitting as shown in the figure below [1]. The figure above illustrates how three random bootstrap samples drawn from an exemplary ten-sample dataset ( X_1,X_2, ..., X_{10} ) and their out-of-bag sample for testing may look like. In practice, Bradley Efron and Robert Tibshirani recommend drawing 50 to 200 bootstrap samples as being sufficient for reliable estimates [2].","title":"Overview"},{"location":"user_guide/evaluate/BootstrapOutOfBag/#references","text":"[1] https://sebastianraschka.com/blog/2016/model-evaluation-selection-part2.html [2] Efron, Bradley, and Robert J. Tibshirani. An introduction to the bootstrap. CRC press, 1994. Management of Data (ACM SIGMOD '97), pages 265-276, 1997.","title":"References"},{"location":"user_guide/evaluate/BootstrapOutOfBag/#example-1-evaluating-the-predictive-performance-of-a-model","text":"The BootstrapOutOfBag class mimics the behavior of scikit-learn's cross-validation classes, e.g., KFold : from mlxtend.evaluate import BootstrapOutOfBag import numpy as np oob = BootstrapOutOfBag(n_splits=3) for train, test in oob.split(np.array([1, 2, 3, 4, 5])): print(train, test) [4 2 1 3 3] [0] [2 4 1 2 1] [0 3] [4 3 3 4 1] [0 2] Consequently, we can use BootstrapOutOfBag objects via the cross_val_score method: from sklearn.datasets import load_iris from sklearn.linear_model import LogisticRegression from sklearn.model_selection import cross_val_score iris = load_iris() X = iris.data y = iris.target lr = LogisticRegression() print(cross_val_score(lr, X, y)) [ 0.96078431 0.92156863 0.95833333] print(cross_val_score(lr, X, y, cv=BootstrapOutOfBag(n_splits=3, random_seed=456))) [ 0.92727273 0.96226415 0.94444444] In practice, it is recommended to run at least 200 iterations, though: print('Mean accuracy: %.1f%%' % np.mean(100*cross_val_score( lr, X, y, cv=BootstrapOutOfBag(n_splits=200, random_seed=456)))) Mean accuracy: 94.8% Using the bootstrap, we can use the percentile method to compute the confidence bounds of the performance estimate. We pick our lower and upper confidence bounds as follows: ACC_{lower} = \\alpha_1th percentile of the ACC_{boot} distribution ACC_{lower} = \\alpha_2th percentile of the ACC_{boot} distribution where \\alpha_1 = \\alpha and \\alpha_2 = 1-\\alpha , and the degree of confidence to compute the 100 \\times (1-2 \\times \\alpha) confidence interval. For instance, to compute a 95% confidence interval, we pick \\alpha=0.025 to obtain the 2.5th and 97.5th percentiles of the b bootstrap samples distribution as the upper and lower confidence bounds. import matplotlib.pyplot as plt %matplotlib inline accuracies = cross_val_score(lr, X, y, cv=BootstrapOutOfBag(n_splits=1000, random_seed=456)) mean = np.mean(accuracies) lower = np.percentile(accuracies, 2.5) upper = np.percentile(accuracies, 97.5) fig, ax = plt.subplots(figsize=(8, 4)) ax.vlines(mean, [0], 40, lw=2.5, linestyle='-', label='mean') ax.vlines(lower, [0], 15, lw=2.5, linestyle='-.', label='CI95 percentile') ax.vlines(upper, [0], 15, lw=2.5, linestyle='-.') ax.hist(accuracies, bins=11, color='#0080ff', edgecolor=\"none\", alpha=0.3) plt.legend(loc='upper left') plt.show()","title":"Example 1 -- Evaluating the predictive performance of a model"},{"location":"user_guide/evaluate/BootstrapOutOfBag/#api","text":"BootstrapOutOfBag(n_splits=200, random_seed=None) Parameters n_splits : int (default=200) Number of bootstrap iterations. Must be larger than 1. random_seed : int (default=None) If int, random_seed is the seed used by the random number generator. Returns train_idx : ndarray The training set indices for that split. test_idx : ndarray The testing set indices for that split. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/BootstrapOutOfBag/","title":"API"},{"location":"user_guide/evaluate/BootstrapOutOfBag/#methods","text":"get_n_splits(X=None, y=None, groups=None) Returns the number of splitting iterations in the cross-validator Parameters X : object Always ignored, exists for compatibility with scikit-learn. y : object Always ignored, exists for compatibility with scikit-learn. groups : object Always ignored, exists for compatibility with scikit-learn. Returns n_splits : int Returns the number of splitting iterations in the cross-validator. split(X, y=None, groups=None) y : array-like or None (default: None) Argument is not used and only included as parameter for compatibility, similar to KFold in scikit-learn. groups : array-like or None (default: None) Argument is not used and only included as parameter for compatibility, similar to KFold in scikit-learn.","title":"Methods"},{"location":"user_guide/evaluate/GroupTimeSeriesSplit/","text":"GroupTimeSeriesSplit: A scikit-learn compatible version of the time series validation with groups A scikit-learn-compatible time series cross-validator that supports non-overlapping groups. from mlxtend.evaluate import GroupTimeSeriesSplit Overview Time series tasks in machine learning require special type of validation, because the time order of the objects is important for a fairer evaluation of an ML model\u2019s quality. Also there can be different time units for splitting the data for different tasks - hours, days, months etc. Here, we use time series validation with support of the groups which can be flexibly configured along with other parameters: Test size Train size Number of splits Gap size Shift size Window type This GroupTimeSeriesSplit implementation is inspired by scikit-learn's TimeSeriesSplit but it has several advantages: GroupTimeSeriesSplit lets you group data as you want before splitting, while TimeSeriesSplit only considers the record level. It can be used for both holdout validation (n_splits=1) and cross-validation (n_splits>=2), whereas TimeSeriesSplit can be used only for the latter case. TimeSeriesSplit uses only an expanding window, while for this implementation you can choose between both rolling and expanding window types. GroupTimeSeriesSplit offers additional control for splitting using an additional shift size parameter. There are several features that need to be taken into account: GroupTimeSeriesSplit is compatible with sklearn-learn API. Numbers or custom non-numeric values can be used as groups However, groups should be consecutive Specifying the test size with either a) the train size or b) the number of splits is required parameters for splitting If full data can\u2019t be used with specific parameters, the most recent data is considered for splitting If splitting is impossible (e.g., there is not enough data to split) using specified parameters, an exception will be raised Before we illustrate the usage of GroupTimeSeriesSplit in the following examples below, let's set up a DummyClassifier that we will reuse in the following sections. Also, let's import the libraries we will be using in the following examples: import numpy as np import pandas as pd from sklearn.dummy import DummyClassifier from sklearn.model_selection import cross_val_score from mlxtend.evaluate.time_series import ( GroupTimeSeriesSplit, plot_splits, print_cv_info, print_split_info, ) Prepare sample data For the following examples, we are creating sample dataset consisting of 16 training data points with corresponding targets. Features and targets Let's assume that we have one numeric feature and target for the binary classification task. data = [[0], [7], [6], [4], [4], [8], [0], [6], [2], [0], [5], [9], [7], [7], [7], [7]] target = [1, 0, 1, 0, 1, 0, 0, 1, 1, 1, 0, 1, 1, 0, 0, 0] X = pd.DataFrame(data, columns=[\"num_feature\"]) y = pd.Series(target, name=\"target\") Group numbers We create 6 different groups so that the first training example belongs to group 0, the next 4 to group 1, and so forth. These groups do not have to be in ascending order (as in this dataset), but they must be consecutive. groups = np.array([0, 1, 1, 1, 1, 2, 2, 2, 3, 3, 4, 4, 5, 5, 5, 5]) groups array([0, 1, 1, 1, 1, 2, 2, 2, 3, 3, 4, 4, 5, 5, 5, 5]) Note that the following i another example of a correct group ordering (not sorted but consecutive): np.array([5, 5, 5, 5, 1, 1, 1, 1, 3, 3, 2, 2, 2, 4, 4, 0]) However, the example below shows an incorrect group ordering (not consecutive), which is not compatible with GroupTimeSeriesSplit : np.array([0, 1, 1, 1, 1, 2, 2, 2, 3, 3, 4, 4, 2, 2, 2, 2]) Group names (months) We will add months as the index according to the specified groups for a more illustrative example. months_map = {i: f\"2021-0{i+1}\" for i in range(6)} months = np.array([months_map[group] for group in groups]) months array(['2021-01', '2021-02', '2021-02', '2021-02', '2021-02', '2021-03', '2021-03', '2021-03', '2021-04', '2021-04', '2021-05', '2021-05', '2021-06', '2021-06', '2021-06', '2021-06'], dtype='<U7') X = X.set_index(months) Example 1 -- Multiple training groups (with train size specified) Let's create a time series splitting with a training dataset that consists of 3 groups. And we will use 1 group for testing. In this case, the number of splits will be calculated automatically since both the training set and the test set sizes are specified. The first 3 rows in the visualization describe how each split is distributed among groups. The groups are visualized in the last row where each color represents different group. cv_args = {\"test_size\": 1, \"train_size\": 3} plot_splits(X, y, groups, **cv_args) print_split_info(X, y, groups, **cv_args) Train indices: [0 1 2 3 4 5 6 7] Test indices: [8 9] Train length: 8 Test length: 2 Train groups: [0 1 1 1 1 2 2 2] Test groups: [3 3] Train group size: 3 Test group size: 1 Train group months: ['2021-01' '2021-02' '2021-02' '2021-02' '2021-02' '2021-03' '2021-03' '2021-03'] Test group months: ['2021-04' '2021-04'] Train indices: [1 2 3 4 5 6 7 8 9] Test indices: [10 11] Train length: 9 Test length: 2 Train groups: [1 1 1 1 2 2 2 3 3] Test groups: [4 4] Train group size: 3 Test group size: 1 Train group months: ['2021-02' '2021-02' '2021-02' '2021-02' '2021-03' '2021-03' '2021-03' '2021-04' '2021-04'] Test group months: ['2021-05' '2021-05'] Train indices: [ 5 6 7 8 9 10 11] Test indices: [12 13 14 15] Train length: 7 Test length: 4 Train groups: [2 2 2 3 3 4 4] Test groups: [5 5 5 5] Train group size: 3 Test group size: 1 Train group months: ['2021-03' '2021-03' '2021-03' '2021-04' '2021-04' '2021-05' '2021-05'] Test group months: ['2021-06' '2021-06' '2021-06' '2021-06'] Please note that if we specify the number of groups for both the training and the test set, the split size is determined automatically, and the number of splits naturally changes with the groups sizes. For example, increasing the number of training groups will naturally result in a lower number of splits as shown below. cv_args = {\"test_size\": 1, \"train_size\": 4} plot_splits(X, y, groups, **cv_args) Usage in CV The example below illustrates how we can use the time series splitter with scikit-learn, i.e., using cross_val_score : cv = GroupTimeSeriesSplit(**cv_args) clf = DummyClassifier(strategy=\"most_frequent\") scores = cross_val_score(clf, X, y, groups=groups, scoring=\"accuracy\", cv=cv) print_cv_info(cv, X, y, groups, clf, scores) Split number: 1 Train true target: [1 0 1 0 1 0 0 1] Train predicted target: [0 0 0 0 0 0 0 0] Test true target: [1 1] Test predicted target: [0 0] Accuracy: 0.0 Split number: 2 Train true target: [0 1 0 1 0 0 1 1 1] Train predicted target: [1 1 1 1 1 1 1 1 1] Test true target: [0 1] Test predicted target: [1 1] Accuracy: 0.5 Split number: 3 Train true target: [0 0 1 1 1 0 1] Train predicted target: [1 1 1 1 1 1 1] Test true target: [1 0 0 0] Test predicted target: [1 1 1 1] Accuracy: 0.25 Example 2 -- Multiple training groups (with number of splits specified) Now let's take a look at an example where we don't specify the number of training groups. Here, we will split the dataset with test size (2 groups) and a specified number of splits (3 groups), which is sufficient for calculating the training size automatically. cv_args = {\"test_size\": 2, \"n_splits\": 3} plot_splits(X, y, groups, **cv_args) print_split_info(X, y, groups, **cv_args) Train indices: [0 1 2 3 4] Test indices: [5 6 7 8 9] Train length: 5 Test length: 5 Train groups: [0 1 1 1 1] Test groups: [2 2 2 3 3] Train group size: 2 Test group size: 2 Train group months: ['2021-01' '2021-02' '2021-02' '2021-02' '2021-02'] Test group months: ['2021-03' '2021-03' '2021-03' '2021-04' '2021-04'] Train indices: [1 2 3 4 5 6 7] Test indices: [ 8 9 10 11] Train length: 7 Test length: 4 Train groups: [1 1 1 1 2 2 2] Test groups: [3 3 4 4] Train group size: 2 Test group size: 2 Train group months: ['2021-02' '2021-02' '2021-02' '2021-02' '2021-03' '2021-03' '2021-03'] Test group months: ['2021-04' '2021-04' '2021-05' '2021-05'] Train indices: [5 6 7 8 9] Test indices: [10 11 12 13 14 15] Train length: 5 Test length: 6 Train groups: [2 2 2 3 3] Test groups: [4 4 5 5 5 5] Train group size: 2 Test group size: 2 Train group months: ['2021-03' '2021-03' '2021-03' '2021-04' '2021-04'] Test group months: ['2021-05' '2021-05' '2021-06' '2021-06' '2021-06' '2021-06'] Usage in CV Again, let's take a look at how this looks in a scikit-learn context using cross_val_score : cv = GroupTimeSeriesSplit(**cv_args) clf = DummyClassifier(strategy=\"most_frequent\") scores = cross_val_score(clf, X, y, groups=groups, scoring=\"accuracy\", cv=cv) print_cv_info(cv, X, y, groups, clf, scores) Split number: 1 Train true target: [1 0 1 0 1] Train predicted target: [1 1 1 1 1] Test true target: [0 0 1 1 1] Test predicted target: [1 1 1 1 1] Accuracy: 0.6 Split number: 2 Train true target: [0 1 0 1 0 0 1] Train predicted target: [0 0 0 0 0 0 0] Test true target: [1 1 0 1] Test predicted target: [0 0 0 0] Accuracy: 0.25 Split number: 3 Train true target: [0 0 1 1 1] Train predicted target: [1 1 1 1 1] Test true target: [0 1 1 0 0 0] Test predicted target: [1 1 1 1 1 1] Accuracy: 0.33 Example 3 -- Defining the gap size between training and test datasets GroupTimeSeriesSplit let's you specify a gap size greater than 1 in order to skip a specified number of groups between training and test folds (the default gap size is 0). In the example below, we use a gap of 1 group to illustrate this. cv_args = {\"test_size\": 1, \"n_splits\": 3, \"gap_size\": 1} plot_splits(X, y, groups, **cv_args) print_split_info(X, y, groups, **cv_args) Train indices: [0 1 2 3 4] Test indices: [8 9] Train length: 5 Test length: 2 Train groups: [0 1 1 1 1] Test groups: [3 3] Train group size: 2 Test group size: 1 Train group months: ['2021-01' '2021-02' '2021-02' '2021-02' '2021-02'] Test group months: ['2021-04' '2021-04'] Train indices: [1 2 3 4 5 6 7] Test indices: [10 11] Train length: 7 Test length: 2 Train groups: [1 1 1 1 2 2 2] Test groups: [4 4] Train group size: 2 Test group size: 1 Train group months: ['2021-02' '2021-02' '2021-02' '2021-02' '2021-03' '2021-03' '2021-03'] Test group months: ['2021-05' '2021-05'] Train indices: [5 6 7 8 9] Test indices: [12 13 14 15] Train length: 5 Test length: 4 Train groups: [2 2 2 3 3] Test groups: [5 5 5 5] Train group size: 2 Test group size: 1 Train group months: ['2021-03' '2021-03' '2021-03' '2021-04' '2021-04'] Test group months: ['2021-06' '2021-06' '2021-06' '2021-06'] Usage in CV The example below shows how this looks like in a scikit-learn context using cross_val_score : cv = GroupTimeSeriesSplit(**cv_args) clf = DummyClassifier(strategy=\"most_frequent\") scores = cross_val_score(clf, X, y, groups=groups, scoring=\"accuracy\", cv=cv) print_cv_info(cv, X, y, groups, clf, scores) Split number: 1 Train true target: [1 0 1 0 1] Train predicted target: [1 1 1 1 1] Test true target: [1 1] Test predicted target: [1 1] Accuracy: 1.0 Split number: 2 Train true target: [0 1 0 1 0 0 1] Train predicted target: [0 0 0 0 0 0 0] Test true target: [0 1] Test predicted target: [0 0] Accuracy: 0.5 Split number: 3 Train true target: [0 0 1 1 1] Train predicted target: [1 1 1 1 1] Test true target: [1 0 0 0] Test predicted target: [1 1 1 1] Accuracy: 0.25 API GroupTimeSeriesSplit(test_size, train_size=None, n_splits=None, gap_size=0, shift_size=1, window_type='rolling') Group time series cross-validator. Parameters test_size : int Size of test dataset. train_size : int (default=None) Size of train dataset. n_splits : int (default=None) Number of the splits. gap_size : int (default=0) Gap size between train and test datasets. shift_size : int (default=1) Step to shift for the next fold. window_type : str (default=\"rolling\") Type of the window. Possible values: \"rolling\", \"expanding\". Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/GroupTimeSeriesSplit/ Methods get_n_splits(X=None, y=None, groups=None) Returns the number of splitting iterations in the cross-validator. Parameters X : object Always ignored, exists for compatibility. y : object Always ignored, exists for compatibility. groups : object Always ignored, exists for compatibility. Returns n_splits : int Returns the number of splitting iterations in the cross-validator. split(X, y=None, groups=None) Generate indices to split data into training and test set. Parameters X : array-like Training data. y : array-like (default=None) Always ignored, exists for compatibility. groups : array-like (default=None) Array with group names or sequence numbers. Yields train : ndarray The training set indices for that split. test : ndarray The testing set indices for that split.","title":"GroupTimeSeriesSplit: A scikit-learn compatible version of the time series validation with groups"},{"location":"user_guide/evaluate/GroupTimeSeriesSplit/#grouptimeseriessplit-a-scikit-learn-compatible-version-of-the-time-series-validation-with-groups","text":"A scikit-learn-compatible time series cross-validator that supports non-overlapping groups. from mlxtend.evaluate import GroupTimeSeriesSplit","title":"GroupTimeSeriesSplit: A scikit-learn compatible version of the time series validation with groups"},{"location":"user_guide/evaluate/GroupTimeSeriesSplit/#overview","text":"Time series tasks in machine learning require special type of validation, because the time order of the objects is important for a fairer evaluation of an ML model\u2019s quality. Also there can be different time units for splitting the data for different tasks - hours, days, months etc. Here, we use time series validation with support of the groups which can be flexibly configured along with other parameters: Test size Train size Number of splits Gap size Shift size Window type This GroupTimeSeriesSplit implementation is inspired by scikit-learn's TimeSeriesSplit but it has several advantages: GroupTimeSeriesSplit lets you group data as you want before splitting, while TimeSeriesSplit only considers the record level. It can be used for both holdout validation (n_splits=1) and cross-validation (n_splits>=2), whereas TimeSeriesSplit can be used only for the latter case. TimeSeriesSplit uses only an expanding window, while for this implementation you can choose between both rolling and expanding window types. GroupTimeSeriesSplit offers additional control for splitting using an additional shift size parameter. There are several features that need to be taken into account: GroupTimeSeriesSplit is compatible with sklearn-learn API. Numbers or custom non-numeric values can be used as groups However, groups should be consecutive Specifying the test size with either a) the train size or b) the number of splits is required parameters for splitting If full data can\u2019t be used with specific parameters, the most recent data is considered for splitting If splitting is impossible (e.g., there is not enough data to split) using specified parameters, an exception will be raised Before we illustrate the usage of GroupTimeSeriesSplit in the following examples below, let's set up a DummyClassifier that we will reuse in the following sections. Also, let's import the libraries we will be using in the following examples: import numpy as np import pandas as pd from sklearn.dummy import DummyClassifier from sklearn.model_selection import cross_val_score from mlxtend.evaluate.time_series import ( GroupTimeSeriesSplit, plot_splits, print_cv_info, print_split_info, )","title":"Overview"},{"location":"user_guide/evaluate/GroupTimeSeriesSplit/#prepare-sample-data","text":"For the following examples, we are creating sample dataset consisting of 16 training data points with corresponding targets.","title":"Prepare sample data"},{"location":"user_guide/evaluate/GroupTimeSeriesSplit/#features-and-targets","text":"Let's assume that we have one numeric feature and target for the binary classification task. data = [[0], [7], [6], [4], [4], [8], [0], [6], [2], [0], [5], [9], [7], [7], [7], [7]] target = [1, 0, 1, 0, 1, 0, 0, 1, 1, 1, 0, 1, 1, 0, 0, 0] X = pd.DataFrame(data, columns=[\"num_feature\"]) y = pd.Series(target, name=\"target\")","title":"Features and targets"},{"location":"user_guide/evaluate/GroupTimeSeriesSplit/#group-numbers","text":"We create 6 different groups so that the first training example belongs to group 0, the next 4 to group 1, and so forth. These groups do not have to be in ascending order (as in this dataset), but they must be consecutive. groups = np.array([0, 1, 1, 1, 1, 2, 2, 2, 3, 3, 4, 4, 5, 5, 5, 5]) groups array([0, 1, 1, 1, 1, 2, 2, 2, 3, 3, 4, 4, 5, 5, 5, 5]) Note that the following i another example of a correct group ordering (not sorted but consecutive): np.array([5, 5, 5, 5, 1, 1, 1, 1, 3, 3, 2, 2, 2, 4, 4, 0]) However, the example below shows an incorrect group ordering (not consecutive), which is not compatible with GroupTimeSeriesSplit : np.array([0, 1, 1, 1, 1, 2, 2, 2, 3, 3, 4, 4, 2, 2, 2, 2])","title":"Group numbers"},{"location":"user_guide/evaluate/GroupTimeSeriesSplit/#group-names-months","text":"We will add months as the index according to the specified groups for a more illustrative example. months_map = {i: f\"2021-0{i+1}\" for i in range(6)} months = np.array([months_map[group] for group in groups]) months array(['2021-01', '2021-02', '2021-02', '2021-02', '2021-02', '2021-03', '2021-03', '2021-03', '2021-04', '2021-04', '2021-05', '2021-05', '2021-06', '2021-06', '2021-06', '2021-06'], dtype='<U7') X = X.set_index(months)","title":"Group names (months)"},{"location":"user_guide/evaluate/GroupTimeSeriesSplit/#example-1-multiple-training-groups-with-train-size-specified","text":"Let's create a time series splitting with a training dataset that consists of 3 groups. And we will use 1 group for testing. In this case, the number of splits will be calculated automatically since both the training set and the test set sizes are specified. The first 3 rows in the visualization describe how each split is distributed among groups. The groups are visualized in the last row where each color represents different group. cv_args = {\"test_size\": 1, \"train_size\": 3} plot_splits(X, y, groups, **cv_args) print_split_info(X, y, groups, **cv_args) Train indices: [0 1 2 3 4 5 6 7] Test indices: [8 9] Train length: 8 Test length: 2 Train groups: [0 1 1 1 1 2 2 2] Test groups: [3 3] Train group size: 3 Test group size: 1 Train group months: ['2021-01' '2021-02' '2021-02' '2021-02' '2021-02' '2021-03' '2021-03' '2021-03'] Test group months: ['2021-04' '2021-04'] Train indices: [1 2 3 4 5 6 7 8 9] Test indices: [10 11] Train length: 9 Test length: 2 Train groups: [1 1 1 1 2 2 2 3 3] Test groups: [4 4] Train group size: 3 Test group size: 1 Train group months: ['2021-02' '2021-02' '2021-02' '2021-02' '2021-03' '2021-03' '2021-03' '2021-04' '2021-04'] Test group months: ['2021-05' '2021-05'] Train indices: [ 5 6 7 8 9 10 11] Test indices: [12 13 14 15] Train length: 7 Test length: 4 Train groups: [2 2 2 3 3 4 4] Test groups: [5 5 5 5] Train group size: 3 Test group size: 1 Train group months: ['2021-03' '2021-03' '2021-03' '2021-04' '2021-04' '2021-05' '2021-05'] Test group months: ['2021-06' '2021-06' '2021-06' '2021-06'] Please note that if we specify the number of groups for both the training and the test set, the split size is determined automatically, and the number of splits naturally changes with the groups sizes. For example, increasing the number of training groups will naturally result in a lower number of splits as shown below. cv_args = {\"test_size\": 1, \"train_size\": 4} plot_splits(X, y, groups, **cv_args)","title":"Example 1 -- Multiple training groups (with train size specified)"},{"location":"user_guide/evaluate/GroupTimeSeriesSplit/#usage-in-cv","text":"The example below illustrates how we can use the time series splitter with scikit-learn, i.e., using cross_val_score : cv = GroupTimeSeriesSplit(**cv_args) clf = DummyClassifier(strategy=\"most_frequent\") scores = cross_val_score(clf, X, y, groups=groups, scoring=\"accuracy\", cv=cv) print_cv_info(cv, X, y, groups, clf, scores) Split number: 1 Train true target: [1 0 1 0 1 0 0 1] Train predicted target: [0 0 0 0 0 0 0 0] Test true target: [1 1] Test predicted target: [0 0] Accuracy: 0.0 Split number: 2 Train true target: [0 1 0 1 0 0 1 1 1] Train predicted target: [1 1 1 1 1 1 1 1 1] Test true target: [0 1] Test predicted target: [1 1] Accuracy: 0.5 Split number: 3 Train true target: [0 0 1 1 1 0 1] Train predicted target: [1 1 1 1 1 1 1] Test true target: [1 0 0 0] Test predicted target: [1 1 1 1] Accuracy: 0.25","title":"Usage in CV"},{"location":"user_guide/evaluate/GroupTimeSeriesSplit/#example-2-multiple-training-groups-with-number-of-splits-specified","text":"Now let's take a look at an example where we don't specify the number of training groups. Here, we will split the dataset with test size (2 groups) and a specified number of splits (3 groups), which is sufficient for calculating the training size automatically. cv_args = {\"test_size\": 2, \"n_splits\": 3} plot_splits(X, y, groups, **cv_args) print_split_info(X, y, groups, **cv_args) Train indices: [0 1 2 3 4] Test indices: [5 6 7 8 9] Train length: 5 Test length: 5 Train groups: [0 1 1 1 1] Test groups: [2 2 2 3 3] Train group size: 2 Test group size: 2 Train group months: ['2021-01' '2021-02' '2021-02' '2021-02' '2021-02'] Test group months: ['2021-03' '2021-03' '2021-03' '2021-04' '2021-04'] Train indices: [1 2 3 4 5 6 7] Test indices: [ 8 9 10 11] Train length: 7 Test length: 4 Train groups: [1 1 1 1 2 2 2] Test groups: [3 3 4 4] Train group size: 2 Test group size: 2 Train group months: ['2021-02' '2021-02' '2021-02' '2021-02' '2021-03' '2021-03' '2021-03'] Test group months: ['2021-04' '2021-04' '2021-05' '2021-05'] Train indices: [5 6 7 8 9] Test indices: [10 11 12 13 14 15] Train length: 5 Test length: 6 Train groups: [2 2 2 3 3] Test groups: [4 4 5 5 5 5] Train group size: 2 Test group size: 2 Train group months: ['2021-03' '2021-03' '2021-03' '2021-04' '2021-04'] Test group months: ['2021-05' '2021-05' '2021-06' '2021-06' '2021-06' '2021-06']","title":"Example 2 -- Multiple training groups (with number of splits specified)"},{"location":"user_guide/evaluate/GroupTimeSeriesSplit/#usage-in-cv_1","text":"Again, let's take a look at how this looks in a scikit-learn context using cross_val_score : cv = GroupTimeSeriesSplit(**cv_args) clf = DummyClassifier(strategy=\"most_frequent\") scores = cross_val_score(clf, X, y, groups=groups, scoring=\"accuracy\", cv=cv) print_cv_info(cv, X, y, groups, clf, scores) Split number: 1 Train true target: [1 0 1 0 1] Train predicted target: [1 1 1 1 1] Test true target: [0 0 1 1 1] Test predicted target: [1 1 1 1 1] Accuracy: 0.6 Split number: 2 Train true target: [0 1 0 1 0 0 1] Train predicted target: [0 0 0 0 0 0 0] Test true target: [1 1 0 1] Test predicted target: [0 0 0 0] Accuracy: 0.25 Split number: 3 Train true target: [0 0 1 1 1] Train predicted target: [1 1 1 1 1] Test true target: [0 1 1 0 0 0] Test predicted target: [1 1 1 1 1 1] Accuracy: 0.33","title":"Usage in CV"},{"location":"user_guide/evaluate/GroupTimeSeriesSplit/#example-3-defining-the-gap-size-between-training-and-test-datasets","text":"GroupTimeSeriesSplit let's you specify a gap size greater than 1 in order to skip a specified number of groups between training and test folds (the default gap size is 0). In the example below, we use a gap of 1 group to illustrate this. cv_args = {\"test_size\": 1, \"n_splits\": 3, \"gap_size\": 1} plot_splits(X, y, groups, **cv_args) print_split_info(X, y, groups, **cv_args) Train indices: [0 1 2 3 4] Test indices: [8 9] Train length: 5 Test length: 2 Train groups: [0 1 1 1 1] Test groups: [3 3] Train group size: 2 Test group size: 1 Train group months: ['2021-01' '2021-02' '2021-02' '2021-02' '2021-02'] Test group months: ['2021-04' '2021-04'] Train indices: [1 2 3 4 5 6 7] Test indices: [10 11] Train length: 7 Test length: 2 Train groups: [1 1 1 1 2 2 2] Test groups: [4 4] Train group size: 2 Test group size: 1 Train group months: ['2021-02' '2021-02' '2021-02' '2021-02' '2021-03' '2021-03' '2021-03'] Test group months: ['2021-05' '2021-05'] Train indices: [5 6 7 8 9] Test indices: [12 13 14 15] Train length: 5 Test length: 4 Train groups: [2 2 2 3 3] Test groups: [5 5 5 5] Train group size: 2 Test group size: 1 Train group months: ['2021-03' '2021-03' '2021-03' '2021-04' '2021-04'] Test group months: ['2021-06' '2021-06' '2021-06' '2021-06']","title":"Example 3 -- Defining the gap size between training and test datasets"},{"location":"user_guide/evaluate/GroupTimeSeriesSplit/#usage-in-cv_2","text":"The example below shows how this looks like in a scikit-learn context using cross_val_score : cv = GroupTimeSeriesSplit(**cv_args) clf = DummyClassifier(strategy=\"most_frequent\") scores = cross_val_score(clf, X, y, groups=groups, scoring=\"accuracy\", cv=cv) print_cv_info(cv, X, y, groups, clf, scores) Split number: 1 Train true target: [1 0 1 0 1] Train predicted target: [1 1 1 1 1] Test true target: [1 1] Test predicted target: [1 1] Accuracy: 1.0 Split number: 2 Train true target: [0 1 0 1 0 0 1] Train predicted target: [0 0 0 0 0 0 0] Test true target: [0 1] Test predicted target: [0 0] Accuracy: 0.5 Split number: 3 Train true target: [0 0 1 1 1] Train predicted target: [1 1 1 1 1] Test true target: [1 0 0 0] Test predicted target: [1 1 1 1] Accuracy: 0.25","title":"Usage in CV"},{"location":"user_guide/evaluate/GroupTimeSeriesSplit/#api","text":"GroupTimeSeriesSplit(test_size, train_size=None, n_splits=None, gap_size=0, shift_size=1, window_type='rolling') Group time series cross-validator. Parameters test_size : int Size of test dataset. train_size : int (default=None) Size of train dataset. n_splits : int (default=None) Number of the splits. gap_size : int (default=0) Gap size between train and test datasets. shift_size : int (default=1) Step to shift for the next fold. window_type : str (default=\"rolling\") Type of the window. Possible values: \"rolling\", \"expanding\". Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/GroupTimeSeriesSplit/","title":"API"},{"location":"user_guide/evaluate/GroupTimeSeriesSplit/#methods","text":"get_n_splits(X=None, y=None, groups=None) Returns the number of splitting iterations in the cross-validator. Parameters X : object Always ignored, exists for compatibility. y : object Always ignored, exists for compatibility. groups : object Always ignored, exists for compatibility. Returns n_splits : int Returns the number of splitting iterations in the cross-validator. split(X, y=None, groups=None) Generate indices to split data into training and test set. Parameters X : array-like Training data. y : array-like (default=None) Always ignored, exists for compatibility. groups : array-like (default=None) Array with group names or sequence numbers. Yields train : ndarray The training set indices for that split. test : ndarray The testing set indices for that split.","title":"Methods"},{"location":"user_guide/evaluate/PredefinedHoldoutSplit/","text":"PredefinedHoldoutSplit: Utility for the holdout method compatible with scikit-learn Split a dataset into a train and validation subset for validation based on user-specified indices. from mlxtend.evaluate import PredefinedHoldoutSplit Overview The PredefinedHoldoutSplit class serves as an alternative to scikit-learn's KFold class, where the PredefinedHoldoutSplit class splits a dataset into training and a validation subsets without rotation, based on validation indices specified by the user. The PredefinedHoldoutSplit can be used as argument for cv parameters in scikit-learn's GridSearchCV etc. For performing a random split, see the related RandomHoldoutSplit class. Example 1 -- Iterating Over a PredefinedHoldoutSplit from mlxtend.evaluate import PredefinedHoldoutSplit from mlxtend.data import iris_data X, y = iris_data() h_iter = PredefinedHoldoutSplit(valid_indices=[0, 1, 99]) cnt = 0 for train_ind, valid_ind in h_iter.split(X, y): cnt += 1 print(cnt) 1 print(train_ind[:5]) print(valid_ind[:5]) [2 3 4 5 6] [ 0 1 99] Example 2 -- PredefinedHoldoutSplit in GridSearch from sklearn.model_selection import GridSearchCV from sklearn.neighbors import KNeighborsClassifier from mlxtend.evaluate import PredefinedHoldoutSplit from mlxtend.data import iris_data X, y = iris_data() params = {'n_neighbors': [1, 2, 3, 4, 5]} grid = GridSearchCV(KNeighborsClassifier(), param_grid=params, cv=PredefinedHoldoutSplit(valid_indices=[0, 1, 99])) grid.fit(X, y) GridSearchCV(cv=<mlxtend.evaluate.holdout.PredefinedHoldoutSplit object at 0x7fb300565610>, estimator=KNeighborsClassifier(), param_grid={'n_neighbors': [1, 2, 3, 4, 5]}) API PredefinedHoldoutSplit(valid_indices) Train/Validation set splitter for sklearn's GridSearchCV etc. Uses user-specified train/validation set indices to split a dataset into train/validation sets using user-defined or random indices. Parameters valid_indices : array-like, shape (num_examples,) Indices of the training examples in the training set to be used for validation. All other indices in the training set are used to for a training subset for model fitting. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/PredefinedHoldoutSplit/ Methods get_n_splits(X=None, y=None, groups=None) Returns the number of splitting iterations in the cross-validator Parameters X : object Always ignored, exists for compatibility. y : object Always ignored, exists for compatibility. groups : object Always ignored, exists for compatibility. Returns n_splits : 1 Returns the number of splitting iterations in the cross-validator. Always returns 1. split(X, y, groups=None) Generate indices to split data into training and test set. Parameters X : array-like, shape (num_examples, num_features) Training data, where num_examples is the number of examples and num_features is the number of features. y : array-like, shape (num_examples,) The target variable for supervised learning problems. Stratification is done based on the y labels. groups : object Always ignored, exists for compatibility. Yields train_index : ndarray The training set indices for that split. valid_index : ndarray The validation set indices for that split.","title":"PredefinedHoldoutSplit: Utility for the holdout method compatible with scikit-learn"},{"location":"user_guide/evaluate/PredefinedHoldoutSplit/#predefinedholdoutsplit-utility-for-the-holdout-method-compatible-with-scikit-learn","text":"Split a dataset into a train and validation subset for validation based on user-specified indices. from mlxtend.evaluate import PredefinedHoldoutSplit","title":"PredefinedHoldoutSplit: Utility for the holdout method compatible with scikit-learn"},{"location":"user_guide/evaluate/PredefinedHoldoutSplit/#overview","text":"The PredefinedHoldoutSplit class serves as an alternative to scikit-learn's KFold class, where the PredefinedHoldoutSplit class splits a dataset into training and a validation subsets without rotation, based on validation indices specified by the user. The PredefinedHoldoutSplit can be used as argument for cv parameters in scikit-learn's GridSearchCV etc. For performing a random split, see the related RandomHoldoutSplit class.","title":"Overview"},{"location":"user_guide/evaluate/PredefinedHoldoutSplit/#example-1-iterating-over-a-predefinedholdoutsplit","text":"from mlxtend.evaluate import PredefinedHoldoutSplit from mlxtend.data import iris_data X, y = iris_data() h_iter = PredefinedHoldoutSplit(valid_indices=[0, 1, 99]) cnt = 0 for train_ind, valid_ind in h_iter.split(X, y): cnt += 1 print(cnt) 1 print(train_ind[:5]) print(valid_ind[:5]) [2 3 4 5 6] [ 0 1 99]","title":"Example 1 -- Iterating Over a PredefinedHoldoutSplit"},{"location":"user_guide/evaluate/PredefinedHoldoutSplit/#example-2-predefinedholdoutsplit-in-gridsearch","text":"from sklearn.model_selection import GridSearchCV from sklearn.neighbors import KNeighborsClassifier from mlxtend.evaluate import PredefinedHoldoutSplit from mlxtend.data import iris_data X, y = iris_data() params = {'n_neighbors': [1, 2, 3, 4, 5]} grid = GridSearchCV(KNeighborsClassifier(), param_grid=params, cv=PredefinedHoldoutSplit(valid_indices=[0, 1, 99])) grid.fit(X, y) GridSearchCV(cv=<mlxtend.evaluate.holdout.PredefinedHoldoutSplit object at 0x7fb300565610>, estimator=KNeighborsClassifier(), param_grid={'n_neighbors': [1, 2, 3, 4, 5]})","title":"Example 2 -- PredefinedHoldoutSplit in GridSearch"},{"location":"user_guide/evaluate/PredefinedHoldoutSplit/#api","text":"PredefinedHoldoutSplit(valid_indices) Train/Validation set splitter for sklearn's GridSearchCV etc. Uses user-specified train/validation set indices to split a dataset into train/validation sets using user-defined or random indices. Parameters valid_indices : array-like, shape (num_examples,) Indices of the training examples in the training set to be used for validation. All other indices in the training set are used to for a training subset for model fitting. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/PredefinedHoldoutSplit/","title":"API"},{"location":"user_guide/evaluate/PredefinedHoldoutSplit/#methods","text":"get_n_splits(X=None, y=None, groups=None) Returns the number of splitting iterations in the cross-validator Parameters X : object Always ignored, exists for compatibility. y : object Always ignored, exists for compatibility. groups : object Always ignored, exists for compatibility. Returns n_splits : 1 Returns the number of splitting iterations in the cross-validator. Always returns 1. split(X, y, groups=None) Generate indices to split data into training and test set. Parameters X : array-like, shape (num_examples, num_features) Training data, where num_examples is the number of examples and num_features is the number of features. y : array-like, shape (num_examples,) The target variable for supervised learning problems. Stratification is done based on the y labels. groups : object Always ignored, exists for compatibility. Yields train_index : ndarray The training set indices for that split. valid_index : ndarray The validation set indices for that split.","title":"Methods"},{"location":"user_guide/evaluate/RandomHoldoutSplit/","text":"RandomHoldoutSplit: split a dataset into a train and validation subset for validation Randomly split a dataset into a train and validation subset for validation. from mlxtend.evaluate import RandomHoldoutSplit Overview The RandomHoldoutSplit class serves as an alternative to scikit-learn's KFold class, where the RandomHoldoutSplit class splits a dataset into training and a validation subsets without rotation. The RandomHoldoutSplit can be used as argument for cv parameters in scikit-learn's GridSearchCV etc. The term \"random\" in RandomHoldoutSplit comes from the fact that the split is specified by the random_seed rather than specifying the training and validation set indices manually as in the PredefinedHoldoutSplit class in mlxtend. Example 1 -- Iterating Over a RandomHoldoutSplit from mlxtend.evaluate import RandomHoldoutSplit from mlxtend.data import iris_data X, y = iris_data() h_iter = RandomHoldoutSplit(valid_size=0.3, random_seed=123) cnt = 0 for train_ind, valid_ind in h_iter.split(X, y): cnt += 1 print(cnt) 1 print(train_ind[:5]) print(valid_ind[:5]) [ 60 16 88 130 6] [ 72 125 80 86 117] Example 2 -- RandomHoldoutSplit in GridSearch from sklearn.model_selection import GridSearchCV from sklearn.neighbors import KNeighborsClassifier from mlxtend.evaluate import RandomHoldoutSplit from mlxtend.data import iris_data X, y = iris_data() params = {'n_neighbors': [1, 2, 3, 4, 5]} grid = GridSearchCV(KNeighborsClassifier(), param_grid=params, cv=RandomHoldoutSplit(valid_size=0.3, random_seed=123)) grid.fit(X, y) GridSearchCV(cv=<mlxtend.evaluate.holdout.RandomHoldoutSplit object at 0x7fae707f6610>, estimator=KNeighborsClassifier(), param_grid={'n_neighbors': [1, 2, 3, 4, 5]}) API RandomHoldoutSplit(valid_size=0.5, random_seed=None, stratify=False) Train/Validation set splitter for sklearn's GridSearchCV etc. Provides train/validation set indices to split a dataset into train/validation sets using random indices. Parameters valid_size : float (default: 0.5) Proportion of examples that being assigned as validation examples. 1- valid_size will then automatically be assigned as training set examples. random_seed : int (default: None) The random seed for splitting the data into training and validation set partitions. stratify : bool (default: False) True or False, whether to perform a stratified split or not Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/RandomHoldoutSplit/ Methods get_n_splits(X=None, y=None, groups=None) Returns the number of splitting iterations in the cross-validator Parameters X : object Always ignored, exists for compatibility. y : object Always ignored, exists for compatibility. groups : object Always ignored, exists for compatibility. Returns n_splits : 1 Returns the number of splitting iterations in the cross-validator. Always returns 1. split(X, y, groups=None) Generate indices to split data into training and test set. Parameters X : array-like, shape (num_examples, num_features) Training data, where num_examples is the number of training examples and num_features is the number of features. y : array-like, shape (num_examples,) The target variable for supervised learning problems. Stratification is done based on the y labels. groups : object Always ignored, exists for compatibility. Yields train_index : ndarray The training set indices for that split. valid_index : ndarray The validation set indices for that split.","title":"RandomHoldoutSplit: split a dataset into a train and validation subset for validation"},{"location":"user_guide/evaluate/RandomHoldoutSplit/#randomholdoutsplit-split-a-dataset-into-a-train-and-validation-subset-for-validation","text":"Randomly split a dataset into a train and validation subset for validation. from mlxtend.evaluate import RandomHoldoutSplit","title":"RandomHoldoutSplit: split a dataset into a train and validation subset for validation"},{"location":"user_guide/evaluate/RandomHoldoutSplit/#overview","text":"The RandomHoldoutSplit class serves as an alternative to scikit-learn's KFold class, where the RandomHoldoutSplit class splits a dataset into training and a validation subsets without rotation. The RandomHoldoutSplit can be used as argument for cv parameters in scikit-learn's GridSearchCV etc. The term \"random\" in RandomHoldoutSplit comes from the fact that the split is specified by the random_seed rather than specifying the training and validation set indices manually as in the PredefinedHoldoutSplit class in mlxtend.","title":"Overview"},{"location":"user_guide/evaluate/RandomHoldoutSplit/#example-1-iterating-over-a-randomholdoutsplit","text":"from mlxtend.evaluate import RandomHoldoutSplit from mlxtend.data import iris_data X, y = iris_data() h_iter = RandomHoldoutSplit(valid_size=0.3, random_seed=123) cnt = 0 for train_ind, valid_ind in h_iter.split(X, y): cnt += 1 print(cnt) 1 print(train_ind[:5]) print(valid_ind[:5]) [ 60 16 88 130 6] [ 72 125 80 86 117]","title":"Example 1 -- Iterating Over a RandomHoldoutSplit"},{"location":"user_guide/evaluate/RandomHoldoutSplit/#example-2-randomholdoutsplit-in-gridsearch","text":"from sklearn.model_selection import GridSearchCV from sklearn.neighbors import KNeighborsClassifier from mlxtend.evaluate import RandomHoldoutSplit from mlxtend.data import iris_data X, y = iris_data() params = {'n_neighbors': [1, 2, 3, 4, 5]} grid = GridSearchCV(KNeighborsClassifier(), param_grid=params, cv=RandomHoldoutSplit(valid_size=0.3, random_seed=123)) grid.fit(X, y) GridSearchCV(cv=<mlxtend.evaluate.holdout.RandomHoldoutSplit object at 0x7fae707f6610>, estimator=KNeighborsClassifier(), param_grid={'n_neighbors': [1, 2, 3, 4, 5]})","title":"Example 2 -- RandomHoldoutSplit in GridSearch"},{"location":"user_guide/evaluate/RandomHoldoutSplit/#api","text":"RandomHoldoutSplit(valid_size=0.5, random_seed=None, stratify=False) Train/Validation set splitter for sklearn's GridSearchCV etc. Provides train/validation set indices to split a dataset into train/validation sets using random indices. Parameters valid_size : float (default: 0.5) Proportion of examples that being assigned as validation examples. 1- valid_size will then automatically be assigned as training set examples. random_seed : int (default: None) The random seed for splitting the data into training and validation set partitions. stratify : bool (default: False) True or False, whether to perform a stratified split or not Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/RandomHoldoutSplit/","title":"API"},{"location":"user_guide/evaluate/RandomHoldoutSplit/#methods","text":"get_n_splits(X=None, y=None, groups=None) Returns the number of splitting iterations in the cross-validator Parameters X : object Always ignored, exists for compatibility. y : object Always ignored, exists for compatibility. groups : object Always ignored, exists for compatibility. Returns n_splits : 1 Returns the number of splitting iterations in the cross-validator. Always returns 1. split(X, y, groups=None) Generate indices to split data into training and test set. Parameters X : array-like, shape (num_examples, num_features) Training data, where num_examples is the number of training examples and num_features is the number of features. y : array-like, shape (num_examples,) The target variable for supervised learning problems. Stratification is done based on the y labels. groups : object Always ignored, exists for compatibility. Yields train_index : ndarray The training set indices for that split. valid_index : ndarray The validation set indices for that split.","title":"Methods"},{"location":"user_guide/evaluate/accuracy_score/","text":"accuracy_score: Computing standard, balanced, and per-class accuracy A function for computing for computing basic classifcation accuracy, per-class accuracy, and average per-class accuracy. from mlxtend.evaluate import accuracy_score Example 1 -- Standard Accuracy The \"overall\" accuracy is defined as the number of correct predictions ( true positives TP and true negatives TN) over all samples n : ACC = \\frac{TP + TN}{n} import numpy as np from mlxtend.evaluate import accuracy_score y_targ = [0, 0, 0, 1, 1, 1, 2, 2, 2] y_pred = [1, 0, 0, 0, 1, 2, 0, 2, 2] accuracy_score(y_targ, y_pred) 0.5555555555555556 Example 2 -- Per-Class Accuracy The per-class accuracy is the accuracy of one class (defined as the pos_label ) versus all remaining datapoints in the dataset. import numpy as np from mlxtend.evaluate import accuracy_score y_targ = [0, 0, 0, 1, 1, 1, 2, 2, 2] y_pred = [1, 0, 0, 0, 1, 2, 0, 2, 2] std_acc = accuracy_score(y_targ, y_pred) bin_acc = accuracy_score(y_targ, y_pred, method='binary', pos_label=1) print(f'Standard accuracy: {std_acc*100:.2f}%') print(f'Class 1 accuracy: {bin_acc*100:.2f}%') Standard accuracy: 55.56% Class 1 accuracy: 66.67% Example 3 -- Average Per-Class Accuracy Overview The \"overall\" accuracy is defined as the number of correct predictions ( true positives TP and true negatives TN) over all samples n : ACC = \\frac{TP + TN}{n} in a binary class setting: In a multi-class setting, we can generalize the computation of the accuracy as the fraction of all true predictions (the diagonal) over all samples n. ACC = \\frac{T}{n} Considering a multi-class problem with 3 classes (C0, C1, C2) let's assume our model made the following predictions: We compute the accuracy as: ACC = \\frac{3 + 50 + 18}{90} \\approx 0.79 Now, in order to compute the average per-class accuracy , we compute the binary accuracy for each class label separately; i.e., if class 1 is the positive class, class 0 and 2 are both considered the negative class. APC\\;ACC = \\frac{83/90 + 71/90 + 78/90}{3} \\approx 0.86 import numpy as np from mlxtend.evaluate import accuracy_score y_targ = [0, 0, 0, 1, 1, 1, 2, 0, 0] y_pred = [1, 0, 0, 0, 1, 2, 0, 2, 1] std_acc = accuracy_score(y_targ, y_pred) bin_acc = accuracy_score(y_targ, y_pred, method='binary', pos_label=1) avg_acc = accuracy_score(y_targ, y_pred, method='average') print(f'Standard accuracy: {std_acc*100:.2f}%') print(f'Class 1 accuracy: {bin_acc*100:.2f}%') print(f'Average per-class accuracy: {avg_acc*100:.2f}%') Standard accuracy: 33.33% Class 1 accuracy: 55.56% Average per-class accuracy: 55.56% References [1] S. Raschka. An overview of general performance metrics of binary classifier systems . Computing Research Repository (CoRR), abs/1410.5330, 2014. [2] Cyril Goutte and Eric Gaussier. A probabilistic interpretation of precision, recall and f-score, with implication for evaluation . In Advances in Information Retrieval, pages 345\u2013359. Springer, 2005. [3] Brian W Matthews. Comparison of the predicted and observed secondary structure of T4 phage lysozyme . Biochimica et Biophysica Acta (BBA)- Protein Structure, 405(2):442\u2013451, 1975. API accuracy_score(y_target, y_predicted, method='standard', pos_label=1, normalize=True) General accuracy function for supervised learning. Parameters y_target : array-like, shape=[n_values] True class labels or target values. y_predicted : array-like, shape=[n_values] Predicted class labels or target values. method : str, 'standard' by default. The chosen method for accuracy computation. If set to 'standard', computes overall accuracy. If set to 'binary', computes accuracy for class pos_label. If set to 'average', computes average per-class (balanced) accuracy. If set to 'balanced', computes the scikit-learn-style balanced accuracy. pos_label : str or int, 1 by default. The class whose accuracy score is to be reported. Used only when method is set to 'binary' normalize : bool, True by default. If True, returns fraction of correctly classified samples. If False, returns number of correctly classified samples. Returns score: float Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/accuracy_score/","title":"accuracy_score: Computing standard, balanced, and per-class accuracy"},{"location":"user_guide/evaluate/accuracy_score/#accuracy_score-computing-standard-balanced-and-per-class-accuracy","text":"A function for computing for computing basic classifcation accuracy, per-class accuracy, and average per-class accuracy. from mlxtend.evaluate import accuracy_score","title":"accuracy_score: Computing standard, balanced, and per-class accuracy"},{"location":"user_guide/evaluate/accuracy_score/#example-1-standard-accuracy","text":"The \"overall\" accuracy is defined as the number of correct predictions ( true positives TP and true negatives TN) over all samples n : ACC = \\frac{TP + TN}{n} import numpy as np from mlxtend.evaluate import accuracy_score y_targ = [0, 0, 0, 1, 1, 1, 2, 2, 2] y_pred = [1, 0, 0, 0, 1, 2, 0, 2, 2] accuracy_score(y_targ, y_pred) 0.5555555555555556","title":"Example 1 -- Standard Accuracy"},{"location":"user_guide/evaluate/accuracy_score/#example-2-per-class-accuracy","text":"The per-class accuracy is the accuracy of one class (defined as the pos_label ) versus all remaining datapoints in the dataset. import numpy as np from mlxtend.evaluate import accuracy_score y_targ = [0, 0, 0, 1, 1, 1, 2, 2, 2] y_pred = [1, 0, 0, 0, 1, 2, 0, 2, 2] std_acc = accuracy_score(y_targ, y_pred) bin_acc = accuracy_score(y_targ, y_pred, method='binary', pos_label=1) print(f'Standard accuracy: {std_acc*100:.2f}%') print(f'Class 1 accuracy: {bin_acc*100:.2f}%') Standard accuracy: 55.56% Class 1 accuracy: 66.67%","title":"Example 2 -- Per-Class Accuracy"},{"location":"user_guide/evaluate/accuracy_score/#example-3-average-per-class-accuracy","text":"","title":"Example 3 -- Average Per-Class Accuracy"},{"location":"user_guide/evaluate/accuracy_score/#overview","text":"The \"overall\" accuracy is defined as the number of correct predictions ( true positives TP and true negatives TN) over all samples n : ACC = \\frac{TP + TN}{n} in a binary class setting: In a multi-class setting, we can generalize the computation of the accuracy as the fraction of all true predictions (the diagonal) over all samples n. ACC = \\frac{T}{n} Considering a multi-class problem with 3 classes (C0, C1, C2) let's assume our model made the following predictions: We compute the accuracy as: ACC = \\frac{3 + 50 + 18}{90} \\approx 0.79 Now, in order to compute the average per-class accuracy , we compute the binary accuracy for each class label separately; i.e., if class 1 is the positive class, class 0 and 2 are both considered the negative class. APC\\;ACC = \\frac{83/90 + 71/90 + 78/90}{3} \\approx 0.86 import numpy as np from mlxtend.evaluate import accuracy_score y_targ = [0, 0, 0, 1, 1, 1, 2, 0, 0] y_pred = [1, 0, 0, 0, 1, 2, 0, 2, 1] std_acc = accuracy_score(y_targ, y_pred) bin_acc = accuracy_score(y_targ, y_pred, method='binary', pos_label=1) avg_acc = accuracy_score(y_targ, y_pred, method='average') print(f'Standard accuracy: {std_acc*100:.2f}%') print(f'Class 1 accuracy: {bin_acc*100:.2f}%') print(f'Average per-class accuracy: {avg_acc*100:.2f}%') Standard accuracy: 33.33% Class 1 accuracy: 55.56% Average per-class accuracy: 55.56%","title":"Overview"},{"location":"user_guide/evaluate/accuracy_score/#references","text":"[1] S. Raschka. An overview of general performance metrics of binary classifier systems . Computing Research Repository (CoRR), abs/1410.5330, 2014. [2] Cyril Goutte and Eric Gaussier. A probabilistic interpretation of precision, recall and f-score, with implication for evaluation . In Advances in Information Retrieval, pages 345\u2013359. Springer, 2005. [3] Brian W Matthews. Comparison of the predicted and observed secondary structure of T4 phage lysozyme . Biochimica et Biophysica Acta (BBA)- Protein Structure, 405(2):442\u2013451, 1975.","title":"References"},{"location":"user_guide/evaluate/accuracy_score/#api","text":"accuracy_score(y_target, y_predicted, method='standard', pos_label=1, normalize=True) General accuracy function for supervised learning. Parameters y_target : array-like, shape=[n_values] True class labels or target values. y_predicted : array-like, shape=[n_values] Predicted class labels or target values. method : str, 'standard' by default. The chosen method for accuracy computation. If set to 'standard', computes overall accuracy. If set to 'binary', computes accuracy for class pos_label. If set to 'average', computes average per-class (balanced) accuracy. If set to 'balanced', computes the scikit-learn-style balanced accuracy. pos_label : str or int, 1 by default. The class whose accuracy score is to be reported. Used only when method is set to 'binary' normalize : bool, True by default. If True, returns fraction of correctly classified samples. If False, returns number of correctly classified samples. Returns score: float Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/accuracy_score/","title":"API"},{"location":"user_guide/evaluate/bias_variance_decomp/","text":"bias_variance_decomp: Bias-variance decomposition for classification and regression losses Bias variance decomposition of machine learning algorithms for various loss functions. from mlxtend.evaluate import bias_variance_decomp Overview Often, researchers use the terms bias and variance or \"bias-variance tradeoff\" to describe the performance of a model -- i.e., you may stumble upon talks, books, or articles where people say that a model has a high variance or high bias. So, what does that mean? In general, we might say that \"high variance\" is proportional to overfitting, and \"high bias\" is proportional to underfitting. Anyways, why are we attempting to do this bias-variance decomposition in the first place? The decomposition of the loss into bias and variance helps us understand learning algorithms, as these concepts are correlated to underfitting and overfitting. To use the more formal terms for bias and variance, assume we have a point estimator \\hat{\\theta} of some parameter or function \\theta . Then, the bias is commonly defined as the difference between the expected value of the estimator and the parameter that we want to estimate: \\text{Bias} = E[\\hat{\\theta}] - \\theta. If the bias is larger than zero, we also say that the estimator is positively biased, if the bias is smaller than zero, the estimator is negatively biased, and if the bias is exactly zero, the estimator is unbiased. Similarly, we define the variance as the difference between the expected value of the squared estimator minus the squared expectation of the estimator: \\text{Var}(\\hat{\\theta}) = E\\big[\\hat{\\theta}^2\\big] - \\bigg(E\\big[\\hat{\\theta}\\big]\\bigg)^2. Note that in the context of this lecture, it will be more convenient to write the variance in its alternative form: \\text{Var}(\\hat{\\theta}) = E[(E[{\\hat{\\theta}}] - \\hat{\\theta})^2]. To illustrate the concept further in context of machine learning ... Suppose there is an unknown target function or \"true function\" to which we do want to approximate. Now, suppose we have different training sets drawn from an unknown distribution defined as \"true function + noise.\" The following plot shows different linear regression models, each fit to a different training set. None of these hypotheses approximate the true function well, except at two points (around x=-10 and x=6). Here, we can say that the bias is large because the difference between the true value and the predicted value, on average (here, average means \"expectation of the training sets\" not \"expectation over examples in the training set\"), is large: The next plot shows different unpruned decision tree models, each fit to a different training set. Note that these hypotheses fit the training data very closely. However, if we would consider the expectation over training sets, the average hypothesis would fit the true function perfectly (given that the noise is unbiased and has an expected value of 0). As we can see, the variance is very large, since on average, a prediction differs a lot from the expectation value of the prediction: Bias-Variance Decomposition of the Squared Loss We can decompose a loss function such as the squared loss into three terms, a variance, bias, and a noise term (and the same is true for the decomposition of the 0-1 loss later). However, for simplicity, we will ignore the noise term. Before we introduce the bias-variance decomposition of the 0-1 loss for classification, let us start with the decomposition of the squared loss as an easy warm-up exercise to get familiar with the overall concept. The previous section already listed the common formal definitions of bias and variance, however, let us define them again for convenience: \\text{Bias}(\\hat{\\theta}) = E[\\hat{\\theta}] - \\theta, \\quad \\text{Var}(\\hat{\\theta}) = E[(E[{\\hat{\\theta}}] - \\hat{\\theta})^2]. Recall that in the context of these machine learning lecture (notes), we defined the true or target function as y = f(x) , the predicted target value as \\hat{y} = \\hat{f}(x) = h(x) , and the squared loss as S = (y - \\hat{y})^2 . (I use S here because it will be easier to tell it apart from the E , which we use for the expectation in this lecture.) Note that unless noted otherwise, the expectation is over training sets! To get started with the squared error loss decomposition into bias and variance, let use do some algebraic manipulation, i.e., adding and subtracting the expected value of \\hat{y} and then expanding the expression using the quadratic formula (a+b)^2 = a^2 + b^2 + 2ab) : \\begin{equation} \\begin{split} S = (y - \\hat{y})^2 \\\\ (y - \\hat{y})^2 &= (y - E[{\\hat{y}}] + E[{\\hat{y}}] - \\hat{y})^2 \\\\ &= (y-E[{\\hat{y}}])^2 + (E[{\\hat{y}}] - y)^2 + 2(y - E[\\hat{y}])(E[\\hat{y}] - \\hat{y}). \\end{split} \\end{equation} Next, we just use the expectation on both sides, and we are already done: \\begin{align} E[S] &= E[(y - \\hat{y})^2] \\\\ E[(y - \\hat{y})^2] &= (y-E[{\\hat{y}}])^2 + E[(E[{\\hat{y}}] - \\hat{y})^2]\\\\ &= \\text{[Bias]}^2 + \\text{Variance}. \\end{align} You may wonder what happened to the \" 2ab \" term ( 2(y - E[\\hat{y}])(E[\\hat{y}] - \\hat{y}) ) when we used the expectation. It turns that it evaluates to zero and hence vanishes from the equation, which can be shown as follows: \\begin{align} E[2(y - E[{\\hat{y}}])(E[{\\hat{y}}] - \\hat{y})] &= 2 E[(y - E[{\\hat{y}}])(E[{\\hat{y}}] - \\hat{y})] \\\\ &= 2(y - E[{\\hat{y}}])E[(E[{\\hat{y}}] - \\hat{y})] \\\\ &= 2(y - E[{\\hat{y}}])(E[E[{\\hat{y}}]] - E[\\hat{y}])\\\\ &= 2(y - E[{\\hat{y}}])(E[{\\hat{y}}] - E[{\\hat{y}}]) \\\\ &= 0. \\end{align} So, this is the canonical decomposition of the squared error loss into bias and variance. The next section will discuss some approaches that have been made to decompose the 0-1 loss that we commonly use for classification accuracy or error. The following figure is a sketch of variance and bias in relation to the training error and generalization error -- how high variance related to overfitting, and how large bias relates to underfitting: Bias-Variance Decomposition of the 0-1 Loss Note that decomposing the 0-1 loss into bias and variance components is not as straight-forward as for the squared error loss. To quote Pedro Domingos, a well-known machine learning researcher and professor at University of Washington: \"several authors have proposed bias-variance decompositions related to zero-one loss (Kong & Dietterich, 1995; Breiman, 1996b; Kohavi & Wolpert, 1996; Tibshirani, 1996; Friedman, 1997). However, each of these decompositions has significant shortcomings.\". [1] In fact, the paper this quote was taken from may offer the most intuitive and general formulation at this point. However, we will first, for simplicity, go over Kong & Dietterich formulation [2] of the 0-1 loss decomposition, which is the same as Domingos's but excluding the noise term (for simplicity). The table below summarizes the relevant terms we used for the squared loss in relation to the 0-1 loss. Recall that the 0-1 loss, L , is 0 if a class label is predicted correctly, and one otherwise. The main prediction for the squared error loss is simply the average over the predictions E[\\hat{y}] (the expectation is over training sets), for the 0-1 loss Kong & Dietterich and Domingos defined it as the mode. I.e., if a model predicts the label one more than 50% of the time (considering all possible training sets), then the main prediction is 1, and 0 otherwise. - Squared Loss 0-1 Loss Single loss (y - \\hat{y})^2 L(y, \\hat{y}) Expected loss E[(y - \\hat{y})^2] E[L(y, \\hat{y})] Main prediction E[\\hat{y}] mean (average) mode Bias ^2 (y-E[{\\hat{y}}])^2 L(y, E[\\hat{y}]) Variance E[(E[{\\hat{y}}] - \\hat{y})^2] E[L(\\hat{y}, E[\\hat{y}])] Hence, as result from using the mode to define the main prediction of the 0-1 loss, the bias is 1 if the main prediction does not agree with the true label y , and 0 otherwise: Bias = \\begin{cases} 1 \\text{ if } y \\neq E[{\\hat{y}}], \\\\ 0 \\text{ otherwise}. \\end{cases} The variance of the 0-1 loss is defined as the probability that the predicted label does not match the main prediction: Variance = P(\\hat{y} \\neq E[\\hat{{y}}]). Next, let us take a look at what happens to the loss if the bias is 0. Given the general definition of the loss, loss = bias + variance, if the bias is 0, then we define the loss as the variance: Loss = 0 + Variance = Loss = P(\\hat{y} \\neq y) = Variance = P(\\hat{y} \\neq E[\\hat{{y}}]). In other words, if a model has zero bias, it's loss is entirely defined by the variance, which is intuitive if we think of variance in the context of being proportional overfitting. The more surprising scenario is if the bias is equal to 1. If the bias is equal to 1, as explained by Pedro Domingos, the increasing the variance can decrease the loss, which is an interesting observation. This can be seen by first rewriting the 0-1 loss function as Loss = P(\\hat{y} \\neq y) = 1 - P(\\hat{y} = y). (Note that we have not done anything new, yet.) Now, if we look at the previous equation of the bias, if the bias is 1, we have y \\neq E[{\\hat{y}}] . If y is not equal to the main prediction, but y is also is equal to \\hat{y} , then \\hat{y} must be equal to the main prediction. Using the \"inverse\" (\"1 minus\"), we can then write the loss as Loss = P(\\hat{y} \\neq y) = 1 - P(\\hat{y} = y) = 1 - P(\\hat{y} \\neq E[{\\hat{y}}]). Since the bias is 1, the loss is hence defined as \"loss = bias - variance\" if the bias is 1 (or \"loss = 1 - variance\"). This might be quite unintuitive at first, but the explanations Kong, Dietterich, and Domingos offer was that if a model has a very high bias such that it main prediction is always wrong, increasing the variance can be beneficial, since increasing the variance would push the decision boundary, which might lead to some correct predictions just by chance then. In other words, for scenarios with high bias, increasing the variance can improve (decrease) the loss! References [1] Domingos, Pedro. \"A unified bias-variance decomposition.\" Proceedings of 17th International Conference on Machine Learning. 2000. [2] Dietterich, Thomas G., and Eun Bae Kong. Machine learning bias, statistical bias, and statistical variance of decision tree algorithms. Technical report, Department of Computer Science, Oregon State University, 1995. Example 1 -- Bias Variance Decomposition of a Decision Tree Classifier from mlxtend.evaluate import bias_variance_decomp from sklearn.tree import DecisionTreeClassifier from mlxtend.data import iris_data from sklearn.model_selection import train_test_split X, y = iris_data() X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=123, shuffle=True, stratify=y) tree = DecisionTreeClassifier(random_state=123) avg_expected_loss, avg_bias, avg_var = bias_variance_decomp( tree, X_train, y_train, X_test, y_test, loss='0-1_loss', random_seed=123) print('Average expected loss: %.3f' % avg_expected_loss) print('Average bias: %.3f' % avg_bias) print('Average variance: %.3f' % avg_var) Average expected loss: 0.062 Average bias: 0.022 Average variance: 0.040 For comparison, the bias-variance decomposition of a bagging classifier, which should intuitively have a lower variance compared than a single decision tree: from sklearn.ensemble import BaggingClassifier tree = DecisionTreeClassifier(random_state=123) bag = BaggingClassifier(base_estimator=tree, n_estimators=100, random_state=123) avg_expected_loss, avg_bias, avg_var = bias_variance_decomp( bag, X_train, y_train, X_test, y_test, loss='0-1_loss', random_seed=123) print('Average expected loss: %.3f' % avg_expected_loss) print('Average bias: %.3f' % avg_bias) print('Average variance: %.3f' % avg_var) Average expected loss: 0.048 Average bias: 0.022 Average variance: 0.026 Example 2 -- Bias Variance Decomposition of a Decision Tree Regressor from mlxtend.evaluate import bias_variance_decomp from sklearn.tree import DecisionTreeRegressor from mlxtend.data import boston_housing_data from sklearn.model_selection import train_test_split X, y = boston_housing_data() X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=123, shuffle=True) tree = DecisionTreeRegressor(random_state=123) avg_expected_loss, avg_bias, avg_var = bias_variance_decomp( tree, X_train, y_train, X_test, y_test, loss='mse', random_seed=123) print('Average expected loss: %.3f' % avg_expected_loss) print('Average bias: %.3f' % avg_bias) print('Average variance: %.3f' % avg_var) Average expected loss: 31.536 Average bias: 14.096 Average variance: 17.440 For comparison, the bias-variance decomposition of a bagging regressor is shown below, which should intuitively have a lower variance than a single decision tree: from sklearn.ensemble import BaggingRegressor tree = DecisionTreeRegressor(random_state=123) bag = BaggingRegressor(base_estimator=tree, n_estimators=100, random_state=123) avg_expected_loss, avg_bias, avg_var = bias_variance_decomp( bag, X_train, y_train, X_test, y_test, loss='mse', random_seed=123) print('Average expected loss: %.3f' % avg_expected_loss) print('Average bias: %.3f' % avg_bias) print('Average variance: %.3f' % avg_var) Average expected loss: 18.620 Average bias: 15.461 Average variance: 3.159 Example 3 -- TensorFlow/Keras Support Since mlxtend v0.18.0, the bias_variance_decomp now supports Keras models. Note that the original model is reset in each round (before refitting it to the bootstrap samples). from mlxtend.evaluate import bias_variance_decomp from mlxtend.data import boston_housing_data from sklearn.model_selection import train_test_split from sklearn.metrics import mean_squared_error import tensorflow as tf import numpy as np np.random.seed(1) tf.random.set_seed(1) X, y = boston_housing_data() X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=123, shuffle=True) model = tf.keras.Sequential([ tf.keras.layers.Dense(32, activation=tf.nn.relu), tf.keras.layers.Dense(1) ]) optimizer = tf.keras.optimizers.Adam() model.compile(loss='mean_squared_error', optimizer=optimizer) model.fit(X_train, y_train, epochs=100, verbose=0) mean_squared_error(model.predict(X_test), y_test) 32.69300595184836 Note that it is highly recommended to use the same number of training epochs that you would use on the original training set to ensure convergence: np.random.seed(1) tf.random.set_seed(1) avg_expected_loss, avg_bias, avg_var = bias_variance_decomp( model, X_train, y_train, X_test, y_test, loss='mse', num_rounds=100, random_seed=123, epochs=200, # fit_param verbose=0) # fit_param print('Average expected loss: %.3f' % avg_expected_loss) print('Average bias: %.3f' % avg_bias) print('Average variance: %.3f' % avg_var) Average expected loss: 32.740 Average bias: 27.474 Average variance: 5.265 API bias_variance_decomp(estimator, X_train, y_train, X_test, y_test, loss='0-1_loss', num_rounds=200, random_seed=None, fit_params) estimator : object A classifier or regressor object or class implementing both a fit and predict method similar to the scikit-learn API. X_train : array-like, shape=(num_examples, num_features) A training dataset for drawing the bootstrap samples to carry out the bias-variance decomposition. y_train : array-like, shape=(num_examples) Targets (class labels, continuous values in case of regression) associated with the X_train examples. X_test : array-like, shape=(num_examples, num_features) The test dataset for computing the average loss, bias, and variance. y_test : array-like, shape=(num_examples) Targets (class labels, continuous values in case of regression) associated with the X_test examples. loss : str (default='0-1_loss') Loss function for performing the bias-variance decomposition. Currently allowed values are '0-1_loss' and 'mse'. num_rounds : int (default=200) Number of bootstrap rounds (sampling from the training set) for performing the bias-variance decomposition. Each bootstrap sample has the same size as the original training set. random_seed : int (default=None) Random seed for the bootstrap sampling used for the bias-variance decomposition. fit_params : additional parameters Additional parameters to be passed to the .fit() function of the estimator when it is fit to the bootstrap samples. Returns avg_expected_loss, avg_bias, avg_var : returns the average expected average bias, and average bias (all floats), where the average is computed over the data points in the test set. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/bias_variance_decomp/","title":"bias_variance_decomp: Bias-variance decomposition for classification and regression losses"},{"location":"user_guide/evaluate/bias_variance_decomp/#bias_variance_decomp-bias-variance-decomposition-for-classification-and-regression-losses","text":"Bias variance decomposition of machine learning algorithms for various loss functions. from mlxtend.evaluate import bias_variance_decomp","title":"bias_variance_decomp: Bias-variance decomposition for classification and regression losses"},{"location":"user_guide/evaluate/bias_variance_decomp/#overview","text":"Often, researchers use the terms bias and variance or \"bias-variance tradeoff\" to describe the performance of a model -- i.e., you may stumble upon talks, books, or articles where people say that a model has a high variance or high bias. So, what does that mean? In general, we might say that \"high variance\" is proportional to overfitting, and \"high bias\" is proportional to underfitting. Anyways, why are we attempting to do this bias-variance decomposition in the first place? The decomposition of the loss into bias and variance helps us understand learning algorithms, as these concepts are correlated to underfitting and overfitting. To use the more formal terms for bias and variance, assume we have a point estimator \\hat{\\theta} of some parameter or function \\theta . Then, the bias is commonly defined as the difference between the expected value of the estimator and the parameter that we want to estimate: \\text{Bias} = E[\\hat{\\theta}] - \\theta. If the bias is larger than zero, we also say that the estimator is positively biased, if the bias is smaller than zero, the estimator is negatively biased, and if the bias is exactly zero, the estimator is unbiased. Similarly, we define the variance as the difference between the expected value of the squared estimator minus the squared expectation of the estimator: \\text{Var}(\\hat{\\theta}) = E\\big[\\hat{\\theta}^2\\big] - \\bigg(E\\big[\\hat{\\theta}\\big]\\bigg)^2. Note that in the context of this lecture, it will be more convenient to write the variance in its alternative form: \\text{Var}(\\hat{\\theta}) = E[(E[{\\hat{\\theta}}] - \\hat{\\theta})^2]. To illustrate the concept further in context of machine learning ... Suppose there is an unknown target function or \"true function\" to which we do want to approximate. Now, suppose we have different training sets drawn from an unknown distribution defined as \"true function + noise.\" The following plot shows different linear regression models, each fit to a different training set. None of these hypotheses approximate the true function well, except at two points (around x=-10 and x=6). Here, we can say that the bias is large because the difference between the true value and the predicted value, on average (here, average means \"expectation of the training sets\" not \"expectation over examples in the training set\"), is large: The next plot shows different unpruned decision tree models, each fit to a different training set. Note that these hypotheses fit the training data very closely. However, if we would consider the expectation over training sets, the average hypothesis would fit the true function perfectly (given that the noise is unbiased and has an expected value of 0). As we can see, the variance is very large, since on average, a prediction differs a lot from the expectation value of the prediction:","title":"Overview"},{"location":"user_guide/evaluate/bias_variance_decomp/#bias-variance-decomposition-of-the-squared-loss","text":"We can decompose a loss function such as the squared loss into three terms, a variance, bias, and a noise term (and the same is true for the decomposition of the 0-1 loss later). However, for simplicity, we will ignore the noise term. Before we introduce the bias-variance decomposition of the 0-1 loss for classification, let us start with the decomposition of the squared loss as an easy warm-up exercise to get familiar with the overall concept. The previous section already listed the common formal definitions of bias and variance, however, let us define them again for convenience: \\text{Bias}(\\hat{\\theta}) = E[\\hat{\\theta}] - \\theta, \\quad \\text{Var}(\\hat{\\theta}) = E[(E[{\\hat{\\theta}}] - \\hat{\\theta})^2]. Recall that in the context of these machine learning lecture (notes), we defined the true or target function as y = f(x) , the predicted target value as \\hat{y} = \\hat{f}(x) = h(x) , and the squared loss as S = (y - \\hat{y})^2 . (I use S here because it will be easier to tell it apart from the E , which we use for the expectation in this lecture.) Note that unless noted otherwise, the expectation is over training sets! To get started with the squared error loss decomposition into bias and variance, let use do some algebraic manipulation, i.e., adding and subtracting the expected value of \\hat{y} and then expanding the expression using the quadratic formula (a+b)^2 = a^2 + b^2 + 2ab) : \\begin{equation} \\begin{split} S = (y - \\hat{y})^2 \\\\ (y - \\hat{y})^2 &= (y - E[{\\hat{y}}] + E[{\\hat{y}}] - \\hat{y})^2 \\\\ &= (y-E[{\\hat{y}}])^2 + (E[{\\hat{y}}] - y)^2 + 2(y - E[\\hat{y}])(E[\\hat{y}] - \\hat{y}). \\end{split} \\end{equation} Next, we just use the expectation on both sides, and we are already done: \\begin{align} E[S] &= E[(y - \\hat{y})^2] \\\\ E[(y - \\hat{y})^2] &= (y-E[{\\hat{y}}])^2 + E[(E[{\\hat{y}}] - \\hat{y})^2]\\\\ &= \\text{[Bias]}^2 + \\text{Variance}. \\end{align} You may wonder what happened to the \" 2ab \" term ( 2(y - E[\\hat{y}])(E[\\hat{y}] - \\hat{y}) ) when we used the expectation. It turns that it evaluates to zero and hence vanishes from the equation, which can be shown as follows: \\begin{align} E[2(y - E[{\\hat{y}}])(E[{\\hat{y}}] - \\hat{y})] &= 2 E[(y - E[{\\hat{y}}])(E[{\\hat{y}}] - \\hat{y})] \\\\ &= 2(y - E[{\\hat{y}}])E[(E[{\\hat{y}}] - \\hat{y})] \\\\ &= 2(y - E[{\\hat{y}}])(E[E[{\\hat{y}}]] - E[\\hat{y}])\\\\ &= 2(y - E[{\\hat{y}}])(E[{\\hat{y}}] - E[{\\hat{y}}]) \\\\ &= 0. \\end{align} So, this is the canonical decomposition of the squared error loss into bias and variance. The next section will discuss some approaches that have been made to decompose the 0-1 loss that we commonly use for classification accuracy or error. The following figure is a sketch of variance and bias in relation to the training error and generalization error -- how high variance related to overfitting, and how large bias relates to underfitting:","title":"Bias-Variance Decomposition of the Squared Loss"},{"location":"user_guide/evaluate/bias_variance_decomp/#bias-variance-decomposition-of-the-0-1-loss","text":"Note that decomposing the 0-1 loss into bias and variance components is not as straight-forward as for the squared error loss. To quote Pedro Domingos, a well-known machine learning researcher and professor at University of Washington: \"several authors have proposed bias-variance decompositions related to zero-one loss (Kong & Dietterich, 1995; Breiman, 1996b; Kohavi & Wolpert, 1996; Tibshirani, 1996; Friedman, 1997). However, each of these decompositions has significant shortcomings.\". [1] In fact, the paper this quote was taken from may offer the most intuitive and general formulation at this point. However, we will first, for simplicity, go over Kong & Dietterich formulation [2] of the 0-1 loss decomposition, which is the same as Domingos's but excluding the noise term (for simplicity). The table below summarizes the relevant terms we used for the squared loss in relation to the 0-1 loss. Recall that the 0-1 loss, L , is 0 if a class label is predicted correctly, and one otherwise. The main prediction for the squared error loss is simply the average over the predictions E[\\hat{y}] (the expectation is over training sets), for the 0-1 loss Kong & Dietterich and Domingos defined it as the mode. I.e., if a model predicts the label one more than 50% of the time (considering all possible training sets), then the main prediction is 1, and 0 otherwise. - Squared Loss 0-1 Loss Single loss (y - \\hat{y})^2 L(y, \\hat{y}) Expected loss E[(y - \\hat{y})^2] E[L(y, \\hat{y})] Main prediction E[\\hat{y}] mean (average) mode Bias ^2 (y-E[{\\hat{y}}])^2 L(y, E[\\hat{y}]) Variance E[(E[{\\hat{y}}] - \\hat{y})^2] E[L(\\hat{y}, E[\\hat{y}])] Hence, as result from using the mode to define the main prediction of the 0-1 loss, the bias is 1 if the main prediction does not agree with the true label y , and 0 otherwise: Bias = \\begin{cases} 1 \\text{ if } y \\neq E[{\\hat{y}}], \\\\ 0 \\text{ otherwise}. \\end{cases} The variance of the 0-1 loss is defined as the probability that the predicted label does not match the main prediction: Variance = P(\\hat{y} \\neq E[\\hat{{y}}]). Next, let us take a look at what happens to the loss if the bias is 0. Given the general definition of the loss, loss = bias + variance, if the bias is 0, then we define the loss as the variance: Loss = 0 + Variance = Loss = P(\\hat{y} \\neq y) = Variance = P(\\hat{y} \\neq E[\\hat{{y}}]). In other words, if a model has zero bias, it's loss is entirely defined by the variance, which is intuitive if we think of variance in the context of being proportional overfitting. The more surprising scenario is if the bias is equal to 1. If the bias is equal to 1, as explained by Pedro Domingos, the increasing the variance can decrease the loss, which is an interesting observation. This can be seen by first rewriting the 0-1 loss function as Loss = P(\\hat{y} \\neq y) = 1 - P(\\hat{y} = y). (Note that we have not done anything new, yet.) Now, if we look at the previous equation of the bias, if the bias is 1, we have y \\neq E[{\\hat{y}}] . If y is not equal to the main prediction, but y is also is equal to \\hat{y} , then \\hat{y} must be equal to the main prediction. Using the \"inverse\" (\"1 minus\"), we can then write the loss as Loss = P(\\hat{y} \\neq y) = 1 - P(\\hat{y} = y) = 1 - P(\\hat{y} \\neq E[{\\hat{y}}]). Since the bias is 1, the loss is hence defined as \"loss = bias - variance\" if the bias is 1 (or \"loss = 1 - variance\"). This might be quite unintuitive at first, but the explanations Kong, Dietterich, and Domingos offer was that if a model has a very high bias such that it main prediction is always wrong, increasing the variance can be beneficial, since increasing the variance would push the decision boundary, which might lead to some correct predictions just by chance then. In other words, for scenarios with high bias, increasing the variance can improve (decrease) the loss!","title":"Bias-Variance Decomposition of the 0-1 Loss"},{"location":"user_guide/evaluate/bias_variance_decomp/#references","text":"[1] Domingos, Pedro. \"A unified bias-variance decomposition.\" Proceedings of 17th International Conference on Machine Learning. 2000. [2] Dietterich, Thomas G., and Eun Bae Kong. Machine learning bias, statistical bias, and statistical variance of decision tree algorithms. Technical report, Department of Computer Science, Oregon State University, 1995.","title":"References"},{"location":"user_guide/evaluate/bias_variance_decomp/#example-1-bias-variance-decomposition-of-a-decision-tree-classifier","text":"from mlxtend.evaluate import bias_variance_decomp from sklearn.tree import DecisionTreeClassifier from mlxtend.data import iris_data from sklearn.model_selection import train_test_split X, y = iris_data() X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=123, shuffle=True, stratify=y) tree = DecisionTreeClassifier(random_state=123) avg_expected_loss, avg_bias, avg_var = bias_variance_decomp( tree, X_train, y_train, X_test, y_test, loss='0-1_loss', random_seed=123) print('Average expected loss: %.3f' % avg_expected_loss) print('Average bias: %.3f' % avg_bias) print('Average variance: %.3f' % avg_var) Average expected loss: 0.062 Average bias: 0.022 Average variance: 0.040 For comparison, the bias-variance decomposition of a bagging classifier, which should intuitively have a lower variance compared than a single decision tree: from sklearn.ensemble import BaggingClassifier tree = DecisionTreeClassifier(random_state=123) bag = BaggingClassifier(base_estimator=tree, n_estimators=100, random_state=123) avg_expected_loss, avg_bias, avg_var = bias_variance_decomp( bag, X_train, y_train, X_test, y_test, loss='0-1_loss', random_seed=123) print('Average expected loss: %.3f' % avg_expected_loss) print('Average bias: %.3f' % avg_bias) print('Average variance: %.3f' % avg_var) Average expected loss: 0.048 Average bias: 0.022 Average variance: 0.026","title":"Example 1 -- Bias Variance Decomposition of a Decision Tree Classifier"},{"location":"user_guide/evaluate/bias_variance_decomp/#example-2-bias-variance-decomposition-of-a-decision-tree-regressor","text":"from mlxtend.evaluate import bias_variance_decomp from sklearn.tree import DecisionTreeRegressor from mlxtend.data import boston_housing_data from sklearn.model_selection import train_test_split X, y = boston_housing_data() X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=123, shuffle=True) tree = DecisionTreeRegressor(random_state=123) avg_expected_loss, avg_bias, avg_var = bias_variance_decomp( tree, X_train, y_train, X_test, y_test, loss='mse', random_seed=123) print('Average expected loss: %.3f' % avg_expected_loss) print('Average bias: %.3f' % avg_bias) print('Average variance: %.3f' % avg_var) Average expected loss: 31.536 Average bias: 14.096 Average variance: 17.440 For comparison, the bias-variance decomposition of a bagging regressor is shown below, which should intuitively have a lower variance than a single decision tree: from sklearn.ensemble import BaggingRegressor tree = DecisionTreeRegressor(random_state=123) bag = BaggingRegressor(base_estimator=tree, n_estimators=100, random_state=123) avg_expected_loss, avg_bias, avg_var = bias_variance_decomp( bag, X_train, y_train, X_test, y_test, loss='mse', random_seed=123) print('Average expected loss: %.3f' % avg_expected_loss) print('Average bias: %.3f' % avg_bias) print('Average variance: %.3f' % avg_var) Average expected loss: 18.620 Average bias: 15.461 Average variance: 3.159","title":"Example 2 -- Bias Variance Decomposition of a Decision Tree Regressor"},{"location":"user_guide/evaluate/bias_variance_decomp/#example-3-tensorflowkeras-support","text":"Since mlxtend v0.18.0, the bias_variance_decomp now supports Keras models. Note that the original model is reset in each round (before refitting it to the bootstrap samples). from mlxtend.evaluate import bias_variance_decomp from mlxtend.data import boston_housing_data from sklearn.model_selection import train_test_split from sklearn.metrics import mean_squared_error import tensorflow as tf import numpy as np np.random.seed(1) tf.random.set_seed(1) X, y = boston_housing_data() X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=123, shuffle=True) model = tf.keras.Sequential([ tf.keras.layers.Dense(32, activation=tf.nn.relu), tf.keras.layers.Dense(1) ]) optimizer = tf.keras.optimizers.Adam() model.compile(loss='mean_squared_error', optimizer=optimizer) model.fit(X_train, y_train, epochs=100, verbose=0) mean_squared_error(model.predict(X_test), y_test) 32.69300595184836 Note that it is highly recommended to use the same number of training epochs that you would use on the original training set to ensure convergence: np.random.seed(1) tf.random.set_seed(1) avg_expected_loss, avg_bias, avg_var = bias_variance_decomp( model, X_train, y_train, X_test, y_test, loss='mse', num_rounds=100, random_seed=123, epochs=200, # fit_param verbose=0) # fit_param print('Average expected loss: %.3f' % avg_expected_loss) print('Average bias: %.3f' % avg_bias) print('Average variance: %.3f' % avg_var) Average expected loss: 32.740 Average bias: 27.474 Average variance: 5.265","title":"Example 3 -- TensorFlow/Keras Support"},{"location":"user_guide/evaluate/bias_variance_decomp/#api","text":"bias_variance_decomp(estimator, X_train, y_train, X_test, y_test, loss='0-1_loss', num_rounds=200, random_seed=None, fit_params) estimator : object A classifier or regressor object or class implementing both a fit and predict method similar to the scikit-learn API. X_train : array-like, shape=(num_examples, num_features) A training dataset for drawing the bootstrap samples to carry out the bias-variance decomposition. y_train : array-like, shape=(num_examples) Targets (class labels, continuous values in case of regression) associated with the X_train examples. X_test : array-like, shape=(num_examples, num_features) The test dataset for computing the average loss, bias, and variance. y_test : array-like, shape=(num_examples) Targets (class labels, continuous values in case of regression) associated with the X_test examples. loss : str (default='0-1_loss') Loss function for performing the bias-variance decomposition. Currently allowed values are '0-1_loss' and 'mse'. num_rounds : int (default=200) Number of bootstrap rounds (sampling from the training set) for performing the bias-variance decomposition. Each bootstrap sample has the same size as the original training set. random_seed : int (default=None) Random seed for the bootstrap sampling used for the bias-variance decomposition. fit_params : additional parameters Additional parameters to be passed to the .fit() function of the estimator when it is fit to the bootstrap samples. Returns avg_expected_loss, avg_bias, avg_var : returns the average expected average bias, and average bias (all floats), where the average is computed over the data points in the test set. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/bias_variance_decomp/","title":"API"},{"location":"user_guide/evaluate/bootstrap/","text":"bootstrap: The ordinary nonparametric boostrap for arbitrary parameters An implementation of the ordinary nonparametric bootstrap to bootstrap a single statistic (for example, the mean. median, R^2 of a regression fit, and so forth). from mlxtend.evaluate import bootstrap Overview The bootstrap offers an easy and effective way to estimate the distribution of a statistic via simulation, by drawing (or generating) new samples from an existing sample with replacement. Note that the bootstrap does not require making any assumptions about the sample statistic or dataset being normally distributed. Using the bootstrap, we can estimate sample statistics and compute the standard error of the mean and confidence intervals as if we have drawn a number of samples from an infinite population. In a nutshell, the bootstrap procedure can be described as follows: Draw a sample with replacement Compute the sample statistic Repeat step 1-2 n times Compute the standard deviation (standard error of the mean of the statistic) Compute the confidence interval Or, in simple terms, we can interpret the bootstrap a means of drawing a potentially endless number of (new) samples from a population by resampling the original dataset. Note that the term \"bootstrap replicate\" is being used quite loosely in current literature; many researchers and practitioners use it to define the number of bootstrap samples we draw from the original dataset. However, in the context of this documentation and the code annotation, we use the original definition of bootstrap repliactes and use it to refer to the statistic computed from a bootstrap sample. References [1] Efron, Bradley, and Robert J. Tibshirani. An introduction to the bootstrap. CRC press, 1994. Management of Data (ACM SIGMOD '97), pages 265-276, 1997. Example 1 -- Bootstrapping the Mean This simple example illustrates how you could bootstrap the mean of a sample. import numpy as np from mlxtend.evaluate import bootstrap rng = np.random.RandomState(123) x = rng.normal(loc=5., size=100) original, std_err, ci_bounds = bootstrap(x, num_rounds=1000, func=np.mean, ci=0.95, seed=123) print('Mean: %.2f, SE: +/- %.2f, CI95: [%.2f, %.2f]' % (original, std_err, ci_bounds[0], ci_bounds[1])) Mean: 5.03, SE: +/- 0.11, CI95: [4.80, 5.26] Example 2 - Bootstrapping a Regression Fit This example illustrates how you can bootstrap the R^2 of a regression fit on the training data. from mlxtend.data import autompg_data from sklearn.linear_model import LinearRegression from sklearn.metrics import r2_score X, y = autompg_data() lr = LinearRegression() def r2_fit(X, model=lr): x, y = X[:, 0].reshape(-1, 1), X[:, 1] pred = lr.fit(x, y).predict(x) return r2_score(y, pred) original, std_err, ci_bounds = bootstrap(X, num_rounds=1000, func=r2_fit, ci=0.95, seed=123) print('Mean: %.2f, SE: +/- %.2f, CI95: [%.2f, %.2f]' % (original, std_err, ci_bounds[0], ci_bounds[1])) Mean: 0.90, SE: +/- 0.01, CI95: [0.89, 0.92] API bootstrap(x, func, num_rounds=1000, ci=0.95, ddof=1, seed=None) Implements the ordinary nonparametric bootstrap Parameters x : NumPy array, shape=(n_samples, [n_columns]) An one or multidimensional array of data records func : A function which computes a statistic that is used to compute the bootstrap replicates (the statistic computed from the bootstrap samples). This function must return a scalar value. For example, np.mean or np.median would be an acceptable argument for func if x is a 1-dimensional array or vector. num_rounds : int (default=1000) The number of bootstrap samnples to draw where each bootstrap sample has the same number of records as the original dataset. ci : int (default=0.95) An integer in the range (0, 1) that represents the confidence level for computing the confidence interval. For example, ci=0.95 (default) will compute the 95% confidence interval from the bootstrap replicates. ddof : int The delta degrees of freedom used when computing the standard error. seed : int or None (default=None) Random seed for generating bootstrap samples. Returns original, standard_error, (lower_ci, upper_ci) : tuple Returns the statistic of the original sample ( original ), the standard error of the estimate, and the respective confidence interval bounds. Examples >>> from mlxtend.evaluate import bootstrap >>> rng = np.random.RandomState(123) >>> x = rng.normal(loc=5., size=100) >>> original, std_err, ci_bounds = bootstrap(x, ... num_rounds=1000, ... func=np.mean, ... ci=0.95, ... seed=123) >>> print('Mean: %.2f, SE: +/- %.2f, CI95: [%.2f, %.2f]' % (original, ... std_err, ... ci_bounds[0], ... ci_bounds[1])) Mean: 5.03, SE: +/- 0.11, CI95: [4.80, 5.26] >>> For more usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/bootstrap/","title":"bootstrap: The ordinary nonparametric boostrap for arbitrary parameters"},{"location":"user_guide/evaluate/bootstrap/#bootstrap-the-ordinary-nonparametric-boostrap-for-arbitrary-parameters","text":"An implementation of the ordinary nonparametric bootstrap to bootstrap a single statistic (for example, the mean. median, R^2 of a regression fit, and so forth). from mlxtend.evaluate import bootstrap","title":"bootstrap: The ordinary nonparametric boostrap for arbitrary parameters"},{"location":"user_guide/evaluate/bootstrap/#overview","text":"The bootstrap offers an easy and effective way to estimate the distribution of a statistic via simulation, by drawing (or generating) new samples from an existing sample with replacement. Note that the bootstrap does not require making any assumptions about the sample statistic or dataset being normally distributed. Using the bootstrap, we can estimate sample statistics and compute the standard error of the mean and confidence intervals as if we have drawn a number of samples from an infinite population. In a nutshell, the bootstrap procedure can be described as follows: Draw a sample with replacement Compute the sample statistic Repeat step 1-2 n times Compute the standard deviation (standard error of the mean of the statistic) Compute the confidence interval Or, in simple terms, we can interpret the bootstrap a means of drawing a potentially endless number of (new) samples from a population by resampling the original dataset. Note that the term \"bootstrap replicate\" is being used quite loosely in current literature; many researchers and practitioners use it to define the number of bootstrap samples we draw from the original dataset. However, in the context of this documentation and the code annotation, we use the original definition of bootstrap repliactes and use it to refer to the statistic computed from a bootstrap sample.","title":"Overview"},{"location":"user_guide/evaluate/bootstrap/#references","text":"[1] Efron, Bradley, and Robert J. Tibshirani. An introduction to the bootstrap. CRC press, 1994. Management of Data (ACM SIGMOD '97), pages 265-276, 1997.","title":"References"},{"location":"user_guide/evaluate/bootstrap/#example-1-bootstrapping-the-mean","text":"This simple example illustrates how you could bootstrap the mean of a sample. import numpy as np from mlxtend.evaluate import bootstrap rng = np.random.RandomState(123) x = rng.normal(loc=5., size=100) original, std_err, ci_bounds = bootstrap(x, num_rounds=1000, func=np.mean, ci=0.95, seed=123) print('Mean: %.2f, SE: +/- %.2f, CI95: [%.2f, %.2f]' % (original, std_err, ci_bounds[0], ci_bounds[1])) Mean: 5.03, SE: +/- 0.11, CI95: [4.80, 5.26]","title":"Example 1 -- Bootstrapping the Mean"},{"location":"user_guide/evaluate/bootstrap/#example-2-bootstrapping-a-regression-fit","text":"This example illustrates how you can bootstrap the R^2 of a regression fit on the training data. from mlxtend.data import autompg_data from sklearn.linear_model import LinearRegression from sklearn.metrics import r2_score X, y = autompg_data() lr = LinearRegression() def r2_fit(X, model=lr): x, y = X[:, 0].reshape(-1, 1), X[:, 1] pred = lr.fit(x, y).predict(x) return r2_score(y, pred) original, std_err, ci_bounds = bootstrap(X, num_rounds=1000, func=r2_fit, ci=0.95, seed=123) print('Mean: %.2f, SE: +/- %.2f, CI95: [%.2f, %.2f]' % (original, std_err, ci_bounds[0], ci_bounds[1])) Mean: 0.90, SE: +/- 0.01, CI95: [0.89, 0.92]","title":"Example 2 - Bootstrapping a Regression Fit"},{"location":"user_guide/evaluate/bootstrap/#api","text":"bootstrap(x, func, num_rounds=1000, ci=0.95, ddof=1, seed=None) Implements the ordinary nonparametric bootstrap Parameters x : NumPy array, shape=(n_samples, [n_columns]) An one or multidimensional array of data records func : A function which computes a statistic that is used to compute the bootstrap replicates (the statistic computed from the bootstrap samples). This function must return a scalar value. For example, np.mean or np.median would be an acceptable argument for func if x is a 1-dimensional array or vector. num_rounds : int (default=1000) The number of bootstrap samnples to draw where each bootstrap sample has the same number of records as the original dataset. ci : int (default=0.95) An integer in the range (0, 1) that represents the confidence level for computing the confidence interval. For example, ci=0.95 (default) will compute the 95% confidence interval from the bootstrap replicates. ddof : int The delta degrees of freedom used when computing the standard error. seed : int or None (default=None) Random seed for generating bootstrap samples. Returns original, standard_error, (lower_ci, upper_ci) : tuple Returns the statistic of the original sample ( original ), the standard error of the estimate, and the respective confidence interval bounds. Examples >>> from mlxtend.evaluate import bootstrap >>> rng = np.random.RandomState(123) >>> x = rng.normal(loc=5., size=100) >>> original, std_err, ci_bounds = bootstrap(x, ... num_rounds=1000, ... func=np.mean, ... ci=0.95, ... seed=123) >>> print('Mean: %.2f, SE: +/- %.2f, CI95: [%.2f, %.2f]' % (original, ... std_err, ... ci_bounds[0], ... ci_bounds[1])) Mean: 5.03, SE: +/- 0.11, CI95: [4.80, 5.26] >>> For more usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/bootstrap/","title":"API"},{"location":"user_guide/evaluate/bootstrap_point632_score/","text":"bootstrap_point632_score: The .632 and .632+ boostrap for classifier evaluation An implementation of the .632 bootstrap to evaluate supervised learning algorithms. from mlxtend.evaluate import bootstrap_point632_score Overview Originally, the bootstrap method aims to determine the statistical properties of an estimator when the underlying distribution was unknown and additional samples are not available. Now, in order to exploit this method for the evaluation of predictive models, such as hypotheses for classification and regression, we may prefer a slightly different approach to bootstrapping using the so-called Out-Of-Bag (OOB) or Leave-One-Out Bootstrap (LOOB) technique. Here, we use out-of-bag samples as test sets for evaluation instead of evaluating the model on the training data. Out-of-bag samples are the unique sets of instances that are not used for model fitting as shown in the figure below [1]. The figure above illustrates how three random bootstrap samples drawn from an exemplary ten-sample dataset ( X_1,X_2, ..., X_{10} ) and their out-of-bag sample for testing may look like. In practice, Bradley Efron and Robert Tibshirani recommend drawing 50 to 200 bootstrap samples as being sufficient for reliable estimates [2]. .632 Bootstrap In 1983, Bradley Efron described the .632 Estimate , a further improvement to address the pessimistic bias of the bootstrap cross-validation approach described above [3]. The pessimistic bias in the \"classic\" bootstrap method can be attributed to the fact that the bootstrap samples only contain approximately 63.2% of the unique samples from the original dataset. For instance, we can compute the probability that a given sample from a dataset of size n is not drawn as a bootstrap sample as P (\\text{not chosen}) = \\bigg(1 - \\frac{1}{n}\\bigg)^n, which is asymptotically equivalent to \\frac{1}{e} \\approx 0.368 as n \\rightarrow \\infty. Vice versa, we can then compute the probability that a sample is chosen as P (\\text{chosen}) = 1 - \\bigg(1 - \\frac{1}{n}\\bigg)^n \\approx 0.632 for reasonably large datasets, so that we'd select approximately 0.632 \\times n uniques samples as bootstrap training sets and reserve 0.368 \\times n out-of-bag samples for testing in each iteration. Now, to address the bias that is due to this the sampling with replacement, Bradley Efron proposed the .632 Estimate that we mentioned earlier, which is computed via the following equation: \\text{ACC}_{boot} = \\frac{1}{b} \\sum_{i=1}^b \\big(0.632 \\cdot \\text{ACC}_{h, i} + 0.368 \\cdot \\text{ACC}_{train}\\big), where \\text{ACC}_{train} is the accuracy computed on the whole training set, and \\text{ACC}_{h, i} is the accuracy on the out-of-bag sample. .632+ Bootstrap Now, while the .632 Boostrap attempts to address the pessimistic bias of the estimate, an optimistic bias may occur with models that tend to overfit so that Bradley Efron and Robert Tibshirani proposed the The .632+ Bootstrap Method (Efron and Tibshirani, 1997). Instead of using a fixed \"weight\" \\omega = 0.632 in ACC_{\\text{boot}} = \\frac{1}{b} \\sum_{i=1}^b \\big(\\omega \\cdot \\text{ACC}_{h, i} + (1-\\omega) \\cdot \\text{ACC}_{train} \\big), we compute the weight \\gamma as \\omega = \\frac{0.632}{1 - 0.368 \\times R}, where R is the relative overfitting rate R = \\frac{(-1) \\times (\\text{ACC}_{h, i} - \\text{ACC}_{train})}{\\gamma - (1 -\\text{ACC}_{h, i})}. (Since we are plugging \\omega into the equation for computing ACC_{boot} that we defined above, \\text{ACC}_{h, i} and \\text{ACC}_{train} still refer to the out-of-bag accuracy in the i th bootstrap round and the whole training set accuracy, respectively.) Further, we need to determine the no-information rate \\gamma in order to compute R . For instance, we can compute \\gamma by fitting a model to a dataset that contains all possible combinations between samples x_{i'} and target class labels y_{i} \u2014 we pretend that the observations and class labels are independent: \\gamma = \\frac{1}{n^2} \\sum_{i=1}^{n} \\sum_{i '=1}^{n} L(y_{i}, f(x_{i '})). Alternatively, we can estimate the no-information rate \\gamma as follows: \\gamma = \\sum_{k=1}^K p_k (1 - q_k), where p_k is the proportion of class k samples observed in the dataset, and q_k is the proportion of class k samples that the classifier predicts in the dataset. References [1] https://sebastianraschka.com/blog/2016/model-evaluation-selection-part2.html [2] Efron, Bradley, and Robert J. Tibshirani. An introduction to the bootstrap. CRC press, 1994. Management of Data (ACM SIGMOD '97), pages 265-276, 1997. [3] Efron, Bradley. 1983. \u201cEstimating the Error Rate of a Prediction Rule: Improvement on Cross-Validation.\u201d Journal of the American Statistical Association 78 (382): 316. doi:10.2307/2288636. [4] Efron, Bradley, and Robert Tibshirani. 1997. \u201cImprovements on Cross-Validation: The .632+ Bootstrap Method.\u201d Journal of the American Statistical Association 92 (438): 548. doi:10.2307/2965703. Example 1 -- Evaluating the predictive performance of a model via the classic out-of-bag Bootstrap The bootstrap_point632_score function mimics the behavior of scikit-learn's `cross_val_score, and a typically usage example is shown below: from sklearn import datasets from sklearn.tree import DecisionTreeClassifier from mlxtend.evaluate import bootstrap_point632_score import numpy as np iris = datasets.load_iris() X = iris.data y = iris.target tree = DecisionTreeClassifier(random_state=123) # Model accuracy scores = bootstrap_point632_score(tree, X, y, method='oob') acc = np.mean(scores) print('Accuracy: %.2f%%' % (100*acc)) # Confidence interval lower = np.percentile(scores, 2.5) upper = np.percentile(scores, 97.5) print('95%% Confidence interval: [%.2f, %.2f]' % (100*lower, 100*upper)) Accuracy: 94.45% 95% Confidence interval: [87.71, 100.00] Example 2 -- Evaluating the predictive performance of a model via the .632 Bootstrap from sklearn import datasets from sklearn.tree import DecisionTreeClassifier from mlxtend.evaluate import bootstrap_point632_score import numpy as np iris = datasets.load_iris() X = iris.data y = iris.target tree = DecisionTreeClassifier(random_state=123) # Model accuracy scores = bootstrap_point632_score(tree, X, y) acc = np.mean(scores) print('Accuracy: %.2f%%' % (100*acc)) # Confidence interval lower = np.percentile(scores, 2.5) upper = np.percentile(scores, 97.5) print('95%% Confidence interval: [%.2f, %.2f]' % (100*lower, 100*upper)) Accuracy: 96.42% 95% Confidence interval: [92.41, 100.00] Example 3 -- Evaluating the predictive performance of a model via the .632+ Bootstrap from sklearn import datasets from sklearn.tree import DecisionTreeClassifier from mlxtend.evaluate import bootstrap_point632_score import numpy as np iris = datasets.load_iris() X = iris.data y = iris.target tree = DecisionTreeClassifier(random_state=123) # Model accuracy scores = bootstrap_point632_score(tree, X, y, method='.632+') acc = np.mean(scores) print('Accuracy: %.2f%%' % (100*acc)) # Confidence interval lower = np.percentile(scores, 2.5) upper = np.percentile(scores, 97.5) print('95%% Confidence interval: [%.2f, %.2f]' % (100*lower, 100*upper)) Accuracy: 96.29% 95% Confidence interval: [91.86, 98.92] API bootstrap_point632_score(estimator, X, y, n_splits=200, method='.632', scoring_func=None, predict_proba=False, random_seed=None, clone_estimator=True) Implementation of the .632 [1] and .632+ [2] bootstrap for supervised learning References: - [1] Efron, Bradley. 1983. \"Estimating the Error Rate of a Prediction Rule: Improvement on Cross-Validation.\" Journal of the American Statistical Association 78 (382): 316. doi:10.2307/2288636. - [2] Efron, Bradley, and Robert Tibshirani. 1997. \"Improvements on Cross-Validation: The .632+ Bootstrap Method.\" Journal of the American Statistical Association 92 (438): 548. doi:10.2307/2965703. Parameters estimator : object An estimator for classification or regression that follows the scikit-learn API and implements \"fit\" and \"predict\" methods. X : array-like The data to fit. Can be, for example a list, or an array at least 2d. y : array-like, optional, default: None The target variable to try to predict in the case of supervised learning. n_splits : int (default=200) Number of bootstrap iterations. Must be larger than 1. method : str (default='.632') The bootstrap method, which can be either - 1) '.632' bootstrap (default) - 2) '.632+' bootstrap - 3) 'oob' (regular out-of-bag, no weighting) for comparison studies. scoring_func : callable, Score function (or loss function) with signature scoring_func(y, y_pred, **kwargs) . If none, uses classification accuracy if the estimator is a classifier and mean squared error if the estimator is a regressor. predict_proba : bool Whether to use the predict_proba function for the estimator argument. This is to be used in conjunction with scoring_func which takes in probability values instead of actual predictions. For example, if the scoring_func is :meth: sklearn.metrics.roc_auc_score , then use predict_proba=True . Note that this requires estimator to have predict_proba method implemented. random_seed : int (default=None) If int, random_seed is the seed used by the random number generator. clone_estimator : bool (default=True) Clones the estimator if true, otherwise fits the original. Returns scores : array of float, shape=(len(list(n_splits)),) Array of scores of the estimator for each bootstrap replicate. Examples >>> from sklearn import datasets, linear_model >>> from mlxtend.evaluate import bootstrap_point632_score >>> iris = datasets.load_iris() >>> X = iris.data >>> y = iris.target >>> lr = linear_model.LogisticRegression() >>> scores = bootstrap_point632_score(lr, X, y) >>> acc = np.mean(scores) >>> print('Accuracy:', acc) 0.953023146884 >>> lower = np.percentile(scores, 2.5) >>> upper = np.percentile(scores, 97.5) >>> print('95%% Confidence interval: [%.2f, %.2f]' % (lower, upper)) 95% Confidence interval: [0.90, 0.98] For more usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/bootstrap_point632_score/","title":"bootstrap_point632_score: The .632 and .632+ boostrap for classifier evaluation"},{"location":"user_guide/evaluate/bootstrap_point632_score/#bootstrap_point632_score-the-632-and-632-boostrap-for-classifier-evaluation","text":"An implementation of the .632 bootstrap to evaluate supervised learning algorithms. from mlxtend.evaluate import bootstrap_point632_score","title":"bootstrap_point632_score: The .632 and .632+ boostrap for classifier evaluation"},{"location":"user_guide/evaluate/bootstrap_point632_score/#overview","text":"Originally, the bootstrap method aims to determine the statistical properties of an estimator when the underlying distribution was unknown and additional samples are not available. Now, in order to exploit this method for the evaluation of predictive models, such as hypotheses for classification and regression, we may prefer a slightly different approach to bootstrapping using the so-called Out-Of-Bag (OOB) or Leave-One-Out Bootstrap (LOOB) technique. Here, we use out-of-bag samples as test sets for evaluation instead of evaluating the model on the training data. Out-of-bag samples are the unique sets of instances that are not used for model fitting as shown in the figure below [1]. The figure above illustrates how three random bootstrap samples drawn from an exemplary ten-sample dataset ( X_1,X_2, ..., X_{10} ) and their out-of-bag sample for testing may look like. In practice, Bradley Efron and Robert Tibshirani recommend drawing 50 to 200 bootstrap samples as being sufficient for reliable estimates [2].","title":"Overview"},{"location":"user_guide/evaluate/bootstrap_point632_score/#632-bootstrap","text":"In 1983, Bradley Efron described the .632 Estimate , a further improvement to address the pessimistic bias of the bootstrap cross-validation approach described above [3]. The pessimistic bias in the \"classic\" bootstrap method can be attributed to the fact that the bootstrap samples only contain approximately 63.2% of the unique samples from the original dataset. For instance, we can compute the probability that a given sample from a dataset of size n is not drawn as a bootstrap sample as P (\\text{not chosen}) = \\bigg(1 - \\frac{1}{n}\\bigg)^n, which is asymptotically equivalent to \\frac{1}{e} \\approx 0.368 as n \\rightarrow \\infty. Vice versa, we can then compute the probability that a sample is chosen as P (\\text{chosen}) = 1 - \\bigg(1 - \\frac{1}{n}\\bigg)^n \\approx 0.632 for reasonably large datasets, so that we'd select approximately 0.632 \\times n uniques samples as bootstrap training sets and reserve 0.368 \\times n out-of-bag samples for testing in each iteration. Now, to address the bias that is due to this the sampling with replacement, Bradley Efron proposed the .632 Estimate that we mentioned earlier, which is computed via the following equation: \\text{ACC}_{boot} = \\frac{1}{b} \\sum_{i=1}^b \\big(0.632 \\cdot \\text{ACC}_{h, i} + 0.368 \\cdot \\text{ACC}_{train}\\big), where \\text{ACC}_{train} is the accuracy computed on the whole training set, and \\text{ACC}_{h, i} is the accuracy on the out-of-bag sample.","title":".632 Bootstrap"},{"location":"user_guide/evaluate/bootstrap_point632_score/#632-bootstrap_1","text":"Now, while the .632 Boostrap attempts to address the pessimistic bias of the estimate, an optimistic bias may occur with models that tend to overfit so that Bradley Efron and Robert Tibshirani proposed the The .632+ Bootstrap Method (Efron and Tibshirani, 1997). Instead of using a fixed \"weight\" \\omega = 0.632 in ACC_{\\text{boot}} = \\frac{1}{b} \\sum_{i=1}^b \\big(\\omega \\cdot \\text{ACC}_{h, i} + (1-\\omega) \\cdot \\text{ACC}_{train} \\big), we compute the weight \\gamma as \\omega = \\frac{0.632}{1 - 0.368 \\times R}, where R is the relative overfitting rate R = \\frac{(-1) \\times (\\text{ACC}_{h, i} - \\text{ACC}_{train})}{\\gamma - (1 -\\text{ACC}_{h, i})}. (Since we are plugging \\omega into the equation for computing ACC_{boot} that we defined above, \\text{ACC}_{h, i} and \\text{ACC}_{train} still refer to the out-of-bag accuracy in the i th bootstrap round and the whole training set accuracy, respectively.) Further, we need to determine the no-information rate \\gamma in order to compute R . For instance, we can compute \\gamma by fitting a model to a dataset that contains all possible combinations between samples x_{i'} and target class labels y_{i} \u2014 we pretend that the observations and class labels are independent: \\gamma = \\frac{1}{n^2} \\sum_{i=1}^{n} \\sum_{i '=1}^{n} L(y_{i}, f(x_{i '})). Alternatively, we can estimate the no-information rate \\gamma as follows: \\gamma = \\sum_{k=1}^K p_k (1 - q_k), where p_k is the proportion of class k samples observed in the dataset, and q_k is the proportion of class k samples that the classifier predicts in the dataset.","title":".632+ Bootstrap"},{"location":"user_guide/evaluate/bootstrap_point632_score/#references","text":"[1] https://sebastianraschka.com/blog/2016/model-evaluation-selection-part2.html [2] Efron, Bradley, and Robert J. Tibshirani. An introduction to the bootstrap. CRC press, 1994. Management of Data (ACM SIGMOD '97), pages 265-276, 1997. [3] Efron, Bradley. 1983. \u201cEstimating the Error Rate of a Prediction Rule: Improvement on Cross-Validation.\u201d Journal of the American Statistical Association 78 (382): 316. doi:10.2307/2288636. [4] Efron, Bradley, and Robert Tibshirani. 1997. \u201cImprovements on Cross-Validation: The .632+ Bootstrap Method.\u201d Journal of the American Statistical Association 92 (438): 548. doi:10.2307/2965703.","title":"References"},{"location":"user_guide/evaluate/bootstrap_point632_score/#example-1-evaluating-the-predictive-performance-of-a-model-via-the-classic-out-of-bag-bootstrap","text":"The bootstrap_point632_score function mimics the behavior of scikit-learn's `cross_val_score, and a typically usage example is shown below: from sklearn import datasets from sklearn.tree import DecisionTreeClassifier from mlxtend.evaluate import bootstrap_point632_score import numpy as np iris = datasets.load_iris() X = iris.data y = iris.target tree = DecisionTreeClassifier(random_state=123) # Model accuracy scores = bootstrap_point632_score(tree, X, y, method='oob') acc = np.mean(scores) print('Accuracy: %.2f%%' % (100*acc)) # Confidence interval lower = np.percentile(scores, 2.5) upper = np.percentile(scores, 97.5) print('95%% Confidence interval: [%.2f, %.2f]' % (100*lower, 100*upper)) Accuracy: 94.45% 95% Confidence interval: [87.71, 100.00]","title":"Example 1 -- Evaluating the predictive performance of a model via the classic out-of-bag Bootstrap"},{"location":"user_guide/evaluate/bootstrap_point632_score/#example-2-evaluating-the-predictive-performance-of-a-model-via-the-632-bootstrap","text":"from sklearn import datasets from sklearn.tree import DecisionTreeClassifier from mlxtend.evaluate import bootstrap_point632_score import numpy as np iris = datasets.load_iris() X = iris.data y = iris.target tree = DecisionTreeClassifier(random_state=123) # Model accuracy scores = bootstrap_point632_score(tree, X, y) acc = np.mean(scores) print('Accuracy: %.2f%%' % (100*acc)) # Confidence interval lower = np.percentile(scores, 2.5) upper = np.percentile(scores, 97.5) print('95%% Confidence interval: [%.2f, %.2f]' % (100*lower, 100*upper)) Accuracy: 96.42% 95% Confidence interval: [92.41, 100.00]","title":"Example 2 -- Evaluating the predictive performance of a model via the .632 Bootstrap"},{"location":"user_guide/evaluate/bootstrap_point632_score/#example-3-evaluating-the-predictive-performance-of-a-model-via-the-632-bootstrap","text":"from sklearn import datasets from sklearn.tree import DecisionTreeClassifier from mlxtend.evaluate import bootstrap_point632_score import numpy as np iris = datasets.load_iris() X = iris.data y = iris.target tree = DecisionTreeClassifier(random_state=123) # Model accuracy scores = bootstrap_point632_score(tree, X, y, method='.632+') acc = np.mean(scores) print('Accuracy: %.2f%%' % (100*acc)) # Confidence interval lower = np.percentile(scores, 2.5) upper = np.percentile(scores, 97.5) print('95%% Confidence interval: [%.2f, %.2f]' % (100*lower, 100*upper)) Accuracy: 96.29% 95% Confidence interval: [91.86, 98.92]","title":"Example 3 -- Evaluating the predictive performance of a model via the .632+ Bootstrap"},{"location":"user_guide/evaluate/bootstrap_point632_score/#api","text":"bootstrap_point632_score(estimator, X, y, n_splits=200, method='.632', scoring_func=None, predict_proba=False, random_seed=None, clone_estimator=True) Implementation of the .632 [1] and .632+ [2] bootstrap for supervised learning References: - [1] Efron, Bradley. 1983. \"Estimating the Error Rate of a Prediction Rule: Improvement on Cross-Validation.\" Journal of the American Statistical Association 78 (382): 316. doi:10.2307/2288636. - [2] Efron, Bradley, and Robert Tibshirani. 1997. \"Improvements on Cross-Validation: The .632+ Bootstrap Method.\" Journal of the American Statistical Association 92 (438): 548. doi:10.2307/2965703. Parameters estimator : object An estimator for classification or regression that follows the scikit-learn API and implements \"fit\" and \"predict\" methods. X : array-like The data to fit. Can be, for example a list, or an array at least 2d. y : array-like, optional, default: None The target variable to try to predict in the case of supervised learning. n_splits : int (default=200) Number of bootstrap iterations. Must be larger than 1. method : str (default='.632') The bootstrap method, which can be either - 1) '.632' bootstrap (default) - 2) '.632+' bootstrap - 3) 'oob' (regular out-of-bag, no weighting) for comparison studies. scoring_func : callable, Score function (or loss function) with signature scoring_func(y, y_pred, **kwargs) . If none, uses classification accuracy if the estimator is a classifier and mean squared error if the estimator is a regressor. predict_proba : bool Whether to use the predict_proba function for the estimator argument. This is to be used in conjunction with scoring_func which takes in probability values instead of actual predictions. For example, if the scoring_func is :meth: sklearn.metrics.roc_auc_score , then use predict_proba=True . Note that this requires estimator to have predict_proba method implemented. random_seed : int (default=None) If int, random_seed is the seed used by the random number generator. clone_estimator : bool (default=True) Clones the estimator if true, otherwise fits the original. Returns scores : array of float, shape=(len(list(n_splits)),) Array of scores of the estimator for each bootstrap replicate. Examples >>> from sklearn import datasets, linear_model >>> from mlxtend.evaluate import bootstrap_point632_score >>> iris = datasets.load_iris() >>> X = iris.data >>> y = iris.target >>> lr = linear_model.LogisticRegression() >>> scores = bootstrap_point632_score(lr, X, y) >>> acc = np.mean(scores) >>> print('Accuracy:', acc) 0.953023146884 >>> lower = np.percentile(scores, 2.5) >>> upper = np.percentile(scores, 97.5) >>> print('95%% Confidence interval: [%.2f, %.2f]' % (lower, upper)) 95% Confidence interval: [0.90, 0.98] For more usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/bootstrap_point632_score/","title":"API"},{"location":"user_guide/evaluate/cochrans_q/","text":"cochrans_q: Cochran's Q test for comparing multiple classifiers Cochran's Q test for comparing the performance of multiple classifiers. from mlxtend.evaluate import cochrans_q Overview Cochran's Q test can be regarded as a generalized version of McNemar's test that can be applied to evaluate multiple classifiers. In a sense, Cochran's Q test is analogous to ANOVA for binary outcomes. To compare more than two classifiers, we can use Cochran's Q test, which has a test statistic Q that is approximately, (similar to McNemar's test), distributed as chi-squared with L-1 degrees of freedom, where L is the number of models we evaluate (since L=2 for McNemar's test, McNemars test statistic approximates a chi-squared distribution with one degree of freedom). More formally, Cochran's Q test tests the hypothesis that there is no difference between the classification accuracies [1]: p_i: H_0 = p_1 = p_2 = \\cdots = p_L. Let \\{D_1, \\dots , D_L\\} be a set of classifiers who have all been tested on the same dataset. If the L classifiers don't perform differently, then the following Q statistic is distributed approximately as \"chi-squared\" with L-1 degrees of freedom: Q_C = (L-1) \\frac{L \\sum^{L}_{i=1}G_{i}^{2} - T^2}{LT - \\sum^{N_{ts}}_{j=1} (L_j)^2}. Here, G_i is the number of objects out of N_{ts} correctly classified by D_i= 1, \\dots L ; L_j is the number of classifiers out of L that correctly classified object \\mathbf{z}_j \\in \\mathbf{Z}_{ts} , where \\mathbf{Z}_{ts} = \\{\\mathbf{z}_1, ... \\mathbf{z}_{N_{ts}}\\} is the test dataset on which the classifers are tested on; and T is the total number of correct number of votes among the L classifiers [2]: T = \\sum_{i=1}^{L} G_i = \\sum^{N_{ts}}_{j=1} L_j. To perform Cochran's Q test, we typically organize the classificier predictions in a binary N_{ts} \\times L matrix. The ij\\text{th} entry of such matrix is 0 if a classifier D_j has misclassified a data example (vector) \\mathbf{z}_i and 1 otherwise (if the classifier predicted the class label l(\\mathbf{z}_i) correctly) [2]. The following example taken from [2] illustrates how the classification results may be organized. For instance, assume we have the ground truth labels of the test dataset y_true and the following predictions by 3 classifiers ( y_model_1 , y_model_2 , and y_model_3 ): y_true = np.array([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]) y_model_1 = np.array([1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]) y_model_2 = np.array([1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]) y_model_3 = np.array([1, 1, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1]) The table of correct (1) and incorrect (0) classifications may then look as follows: D_1 (model 1) D_2 (model 2) D_3 (model 3) Occurrences 1 1 1 80 1 1 0 2 1 0 1 0 1 0 0 2 0 1 1 9 0 1 0 1 0 0 1 3 0 0 0 3 Accuracy 84/100*100% = 84% 92/100*100% = 92% 92/100*100% = 92% By plugging in the respective value into the previous equation, we obtain the following Q value [2]: Q_c = 2 \\times \\frac{3 \\times (84^2 + 92^2 + 92^2) - 268^2}{3\\times 268-(80 \\times 9 + 11 \\times 4 + 6 \\times 1)} \\approx 7.5294. (Note that the Q value in [2] is listed as 3.7647 due to a typo as discussed with the author, the value 7.5294 is the correct one.) Now, the Q value (approximating \\chi^2 ) corresponds to a p-value of approx. 0.023 assuming a \\chi^2 distribution with L-1 = 2 degrees of freedom. Assuming that we chose a significance level of \\alpha=0.05 , we would reject the null hypothesis that all classifiers perform equally well, since 0.023 < \\alpha . In practice, if we successfully rejected the null hypothesis, we could perform multiple post hoc pair-wise tests -- for example, McNemar tests with a Bonferroni correction -- to determine which pairs have different population proportions. References [1] Fleiss, Joseph L., Bruce Levin, and Myunghee Cho Paik. Statistical methods for rates and proportions. John Wiley & Sons, 2013. [2] Kuncheva, Ludmila I. Combining pattern classifiers: methods and algorithms. John Wiley & Sons, 2004. Example 1 - Cochran's Q test import numpy as np from mlxtend.evaluate import cochrans_q from mlxtend.evaluate import mcnemar_table from mlxtend.evaluate import mcnemar ## Dataset: # ground truth labels of the test dataset: y_true = np.array([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]) # predictions by 3 classifiers (`y_model_1`, `y_model_2`, and `y_model_3`): y_model_1 = np.array([1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]) y_model_2 = np.array([1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]) y_model_3 = np.array([1, 1, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1]) Assuming a significance level \\alpha=0.05 , we can conduct Cochran's Q test as follows, to test the null hypothesis there is no difference between the classification accuracies, p_i: H_0 = p_1 = p_2 = \\cdots = p_L : q, p_value = cochrans_q(y_true, y_model_1, y_model_2, y_model_3) print('Q: %.3f' % q) print('p-value: %.3f' % p_value) Q: 7.529 p-value: 0.023 Since the p-value is smaller than \\alpha , we can reject the null hypothesis and conclude that there is a difference between the classification accuracies. As mentioned in the introduction earlier, we could now perform multiple post hoc pair-wise tests -- for example, McNemar tests with a Bonferroni correction -- to determine which pairs have different population proportions. Lastly, let's illustrate that Cochran's Q test is indeed just a generalized version of McNemar's test: chi2, p_value = cochrans_q(y_true, y_model_1, y_model_2) print('Cochran\\'s Q Chi^2: %.3f' % chi2) print('Cochran\\'s Q p-value: %.3f' % p_value) Cochran's Q Chi^2: 5.333 Cochran's Q p-value: 0.021 chi2, p_value = mcnemar(mcnemar_table(y_true, y_model_1, y_model_2), corrected=False) print('McNemar\\'s Chi^2: %.3f' % chi2) print('McNemar\\'s p-value: %.3f' % p_value) McNemar's Chi^2: 5.333 McNemar's p-value: 0.021 API cochrans_q(y_target, y_model_predictions)* Cochran's Q test to compare 2 or more models. Parameters y_target : array-like, shape=[n_samples] True class labels as 1D NumPy array. *y_model_predictions : array-likes, shape=[n_samples] Variable number of 2 or more arrays that contain the predicted class labels from models as 1D NumPy array. Returns q, p : float or None, float Returns the Q (chi-squared) value and the p-value Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/cochrans_q/","title":"cochrans_q: Cochran's Q test for comparing multiple classifiers"},{"location":"user_guide/evaluate/cochrans_q/#cochrans_q-cochrans-q-test-for-comparing-multiple-classifiers","text":"Cochran's Q test for comparing the performance of multiple classifiers. from mlxtend.evaluate import cochrans_q","title":"cochrans_q: Cochran's Q test for comparing multiple classifiers"},{"location":"user_guide/evaluate/cochrans_q/#overview","text":"Cochran's Q test can be regarded as a generalized version of McNemar's test that can be applied to evaluate multiple classifiers. In a sense, Cochran's Q test is analogous to ANOVA for binary outcomes. To compare more than two classifiers, we can use Cochran's Q test, which has a test statistic Q that is approximately, (similar to McNemar's test), distributed as chi-squared with L-1 degrees of freedom, where L is the number of models we evaluate (since L=2 for McNemar's test, McNemars test statistic approximates a chi-squared distribution with one degree of freedom). More formally, Cochran's Q test tests the hypothesis that there is no difference between the classification accuracies [1]: p_i: H_0 = p_1 = p_2 = \\cdots = p_L. Let \\{D_1, \\dots , D_L\\} be a set of classifiers who have all been tested on the same dataset. If the L classifiers don't perform differently, then the following Q statistic is distributed approximately as \"chi-squared\" with L-1 degrees of freedom: Q_C = (L-1) \\frac{L \\sum^{L}_{i=1}G_{i}^{2} - T^2}{LT - \\sum^{N_{ts}}_{j=1} (L_j)^2}. Here, G_i is the number of objects out of N_{ts} correctly classified by D_i= 1, \\dots L ; L_j is the number of classifiers out of L that correctly classified object \\mathbf{z}_j \\in \\mathbf{Z}_{ts} , where \\mathbf{Z}_{ts} = \\{\\mathbf{z}_1, ... \\mathbf{z}_{N_{ts}}\\} is the test dataset on which the classifers are tested on; and T is the total number of correct number of votes among the L classifiers [2]: T = \\sum_{i=1}^{L} G_i = \\sum^{N_{ts}}_{j=1} L_j. To perform Cochran's Q test, we typically organize the classificier predictions in a binary N_{ts} \\times L matrix. The ij\\text{th} entry of such matrix is 0 if a classifier D_j has misclassified a data example (vector) \\mathbf{z}_i and 1 otherwise (if the classifier predicted the class label l(\\mathbf{z}_i) correctly) [2]. The following example taken from [2] illustrates how the classification results may be organized. For instance, assume we have the ground truth labels of the test dataset y_true and the following predictions by 3 classifiers ( y_model_1 , y_model_2 , and y_model_3 ): y_true = np.array([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]) y_model_1 = np.array([1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]) y_model_2 = np.array([1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]) y_model_3 = np.array([1, 1, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1]) The table of correct (1) and incorrect (0) classifications may then look as follows: D_1 (model 1) D_2 (model 2) D_3 (model 3) Occurrences 1 1 1 80 1 1 0 2 1 0 1 0 1 0 0 2 0 1 1 9 0 1 0 1 0 0 1 3 0 0 0 3 Accuracy 84/100*100% = 84% 92/100*100% = 92% 92/100*100% = 92% By plugging in the respective value into the previous equation, we obtain the following Q value [2]: Q_c = 2 \\times \\frac{3 \\times (84^2 + 92^2 + 92^2) - 268^2}{3\\times 268-(80 \\times 9 + 11 \\times 4 + 6 \\times 1)} \\approx 7.5294. (Note that the Q value in [2] is listed as 3.7647 due to a typo as discussed with the author, the value 7.5294 is the correct one.) Now, the Q value (approximating \\chi^2 ) corresponds to a p-value of approx. 0.023 assuming a \\chi^2 distribution with L-1 = 2 degrees of freedom. Assuming that we chose a significance level of \\alpha=0.05 , we would reject the null hypothesis that all classifiers perform equally well, since 0.023 < \\alpha . In practice, if we successfully rejected the null hypothesis, we could perform multiple post hoc pair-wise tests -- for example, McNemar tests with a Bonferroni correction -- to determine which pairs have different population proportions.","title":"Overview"},{"location":"user_guide/evaluate/cochrans_q/#references","text":"[1] Fleiss, Joseph L., Bruce Levin, and Myunghee Cho Paik. Statistical methods for rates and proportions. John Wiley & Sons, 2013. [2] Kuncheva, Ludmila I. Combining pattern classifiers: methods and algorithms. John Wiley & Sons, 2004.","title":"References"},{"location":"user_guide/evaluate/cochrans_q/#example-1-cochrans-q-test","text":"import numpy as np from mlxtend.evaluate import cochrans_q from mlxtend.evaluate import mcnemar_table from mlxtend.evaluate import mcnemar ## Dataset: # ground truth labels of the test dataset: y_true = np.array([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]) # predictions by 3 classifiers (`y_model_1`, `y_model_2`, and `y_model_3`): y_model_1 = np.array([1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]) y_model_2 = np.array([1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]) y_model_3 = np.array([1, 1, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1]) Assuming a significance level \\alpha=0.05 , we can conduct Cochran's Q test as follows, to test the null hypothesis there is no difference between the classification accuracies, p_i: H_0 = p_1 = p_2 = \\cdots = p_L : q, p_value = cochrans_q(y_true, y_model_1, y_model_2, y_model_3) print('Q: %.3f' % q) print('p-value: %.3f' % p_value) Q: 7.529 p-value: 0.023 Since the p-value is smaller than \\alpha , we can reject the null hypothesis and conclude that there is a difference between the classification accuracies. As mentioned in the introduction earlier, we could now perform multiple post hoc pair-wise tests -- for example, McNemar tests with a Bonferroni correction -- to determine which pairs have different population proportions. Lastly, let's illustrate that Cochran's Q test is indeed just a generalized version of McNemar's test: chi2, p_value = cochrans_q(y_true, y_model_1, y_model_2) print('Cochran\\'s Q Chi^2: %.3f' % chi2) print('Cochran\\'s Q p-value: %.3f' % p_value) Cochran's Q Chi^2: 5.333 Cochran's Q p-value: 0.021 chi2, p_value = mcnemar(mcnemar_table(y_true, y_model_1, y_model_2), corrected=False) print('McNemar\\'s Chi^2: %.3f' % chi2) print('McNemar\\'s p-value: %.3f' % p_value) McNemar's Chi^2: 5.333 McNemar's p-value: 0.021","title":"Example 1 - Cochran's Q test"},{"location":"user_guide/evaluate/cochrans_q/#api","text":"cochrans_q(y_target, y_model_predictions)* Cochran's Q test to compare 2 or more models. Parameters y_target : array-like, shape=[n_samples] True class labels as 1D NumPy array. *y_model_predictions : array-likes, shape=[n_samples] Variable number of 2 or more arrays that contain the predicted class labels from models as 1D NumPy array. Returns q, p : float or None, float Returns the Q (chi-squared) value and the p-value Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/cochrans_q/","title":"API"},{"location":"user_guide/evaluate/combined_ftest_5x2cv/","text":"combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons 5x2cv combined F test procedure to compare the performance of two models from mlxtend.evaluate import combined_ftest_5x2cv Overview The 5x2cv combined F test is a procedure for comparing the performance of two models (classifiers or regressors) that was proposed by Alpaydin [1] as a more robust alternative to Dietterich's 5x2cv paired t-test procedure [2]. paired_ttest_5x2cv.md . Dietterich's 5x2cv method was in turn designed to address shortcomings in other methods such as the resampled paired t test (see paired_ttest_resampled ) and the k-fold cross-validated paired t test (see paired_ttest_kfold_cv ). To explain how this method works, let's consider to estimator (e.g., classifiers) A and B. Further, we have a labeled dataset D . In the common hold-out method, we typically split the dataset into 2 parts: a training and a test set. In the 5x2cv paired t test, we repeat the splitting (50% training and 50% test data) 5 times. In each of the 5 iterations, we fit A and B to the training split and evaluate their performance ( p_A and p_B ) on the test split. Then, we rotate the training and test sets (the training set becomes the test set and vice versa) compute the performance again, which results in 2 performance difference measures: p^{(1)} = p^{(1)}_A - p^{(1)}_B and p^{(2)} = p^{(2)}_A - p^{(2)}_B. Then, we estimate mean and variance of the differences: \\overline{p} = \\frac{p^{(1)} + p^{(2)}}{2} and s^2 = (p^{(1)} - \\overline{p})^2 + (p^{(2)} - \\overline{p})^2. The F-statistic proposed by Alpaydin (see paper for justifications) is then computed as \\mathcal{f} = \\frac{\\sum_{i=1}^{5} \\sum_{j=1}^2 (p_i^{j})^2}{2 \\sum_{i=1}^5 s_i^2}, which is approximately F distributed with 10 and 5 degress of freedom. Using the f statistic, the p value can be computed and compared with a previously chosen significance level, e.g., \\alpha=0.05 . If the p value is smaller than \\alpha , we reject the null hypothesis and accept that there is a significant difference in the two models. References [1] Alpaydin, E. (1999). Combined 5\u00d72 cv F test for comparing supervised classification learning algorithms. Neural computation, 11(8), 1885-1892. [2] Dietterich TG (1998) Approximate Statistical Tests for Comparing Supervised Classification Learning Algorithms. Neural Comput 10:1895\u20131923. Example 1 - 5x2cv combined F test Assume we want to compare two classification algorithms, logistic regression and a decision tree algorithm: from sklearn.linear_model import LogisticRegression from sklearn.tree import DecisionTreeClassifier from mlxtend.data import iris_data from sklearn.model_selection import train_test_split X, y = iris_data() clf1 = LogisticRegression(random_state=1, solver='liblinear', multi_class='ovr') clf2 = DecisionTreeClassifier(random_state=1) X_train, X_test, y_train, y_test = \\ train_test_split(X, y, test_size=0.25, random_state=123) score1 = clf1.fit(X_train, y_train).score(X_test, y_test) score2 = clf2.fit(X_train, y_train).score(X_test, y_test) print('Logistic regression accuracy: %.2f%%' % (score1*100)) print('Decision tree accuracy: %.2f%%' % (score2*100)) Logistic regression accuracy: 97.37% Decision tree accuracy: 94.74% Note that these accuracy values are not used in the paired f test procedure as new test/train splits are generated during the resampling procedure, the values above are just serving the purpose of intuition. Now, let's assume a significance threshold of \\alpha=0.05 for rejecting the null hypothesis that both algorithms perform equally well on the dataset and conduct the 5x2cv f test: from mlxtend.evaluate import combined_ftest_5x2cv f, p = combined_ftest_5x2cv(estimator1=clf1, estimator2=clf2, X=X, y=y, random_seed=1) print('F statistic: %.3f' % f) print('p value: %.3f' % p) F statistic: 1.053 p value: 0.509 Since p > \\alpha , we cannot reject the null hypothesis and may conclude that the performance of the two algorithms is not significantly different. While it is generally not recommended to apply statistical tests multiple times without correction for multiple hypothesis testing, let us take a look at an example where the decision tree algorithm is limited to producing a very simple decision boundary that would result in a relatively bad performance: clf2 = DecisionTreeClassifier(random_state=1, max_depth=1) score2 = clf2.fit(X_train, y_train).score(X_test, y_test) print('Decision tree accuracy: %.2f%%' % (score2*100)) f, p = combined_ftest_5x2cv(estimator1=clf1, estimator2=clf2, X=X, y=y, random_seed=1) print('F statistic: %.3f' % f) print('p value: %.3f' % p) Decision tree accuracy: 63.16% F statistic: 34.934 p value: 0.001 Assuming that we conducted this test also with a significance level of \\alpha=0.05 , we can reject the null-hypothesis that both models perform equally well on this dataset, since the p-value ( p < 0.001 ) is smaller than \\alpha . API combined_ftest_5x2cv(estimator1, estimator2, X, y, scoring=None, random_seed=None) Implements the 5x2cv combined F test proposed by Alpaydin 1999, to compare the performance of two models. Parameters estimator1 : scikit-learn classifier or regressor estimator2 : scikit-learn classifier or regressor X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values. scoring : str, callable, or None (default: None) If None (default), uses 'accuracy' for sklearn classifiers and 'r2' for sklearn regressors. If str, uses a sklearn scoring metric string identifier, for example {accuracy, f1, precision, recall, roc_auc} for classifiers, {'mean_absolute_error', 'mean_squared_error'/'neg_mean_squared_error', 'median_absolute_error', 'r2'} for regressors. If a callable object or function is provided, it has to be conform with sklearn's signature scorer(estimator, X, y) ; see https://scikit-learn.org/stable/modules/generated/sklearn.metrics.make_scorer.html for more information. random_seed : int or None (default: None) Random seed for creating the test/train splits. Returns f : float The F-statistic pvalue : float Two-tailed p-value. If the chosen significance level is larger than the p-value, we reject the null hypothesis and accept that there are significant differences in the two compared models. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/combined_ftest_5x2cv/","title":"combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons"},{"location":"user_guide/evaluate/combined_ftest_5x2cv/#combined_ftest_5x2cv-5x2cv-combined-f-test-for-classifier-comparisons","text":"5x2cv combined F test procedure to compare the performance of two models from mlxtend.evaluate import combined_ftest_5x2cv","title":"combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons"},{"location":"user_guide/evaluate/combined_ftest_5x2cv/#overview","text":"The 5x2cv combined F test is a procedure for comparing the performance of two models (classifiers or regressors) that was proposed by Alpaydin [1] as a more robust alternative to Dietterich's 5x2cv paired t-test procedure [2]. paired_ttest_5x2cv.md . Dietterich's 5x2cv method was in turn designed to address shortcomings in other methods such as the resampled paired t test (see paired_ttest_resampled ) and the k-fold cross-validated paired t test (see paired_ttest_kfold_cv ). To explain how this method works, let's consider to estimator (e.g., classifiers) A and B. Further, we have a labeled dataset D . In the common hold-out method, we typically split the dataset into 2 parts: a training and a test set. In the 5x2cv paired t test, we repeat the splitting (50% training and 50% test data) 5 times. In each of the 5 iterations, we fit A and B to the training split and evaluate their performance ( p_A and p_B ) on the test split. Then, we rotate the training and test sets (the training set becomes the test set and vice versa) compute the performance again, which results in 2 performance difference measures: p^{(1)} = p^{(1)}_A - p^{(1)}_B and p^{(2)} = p^{(2)}_A - p^{(2)}_B. Then, we estimate mean and variance of the differences: \\overline{p} = \\frac{p^{(1)} + p^{(2)}}{2} and s^2 = (p^{(1)} - \\overline{p})^2 + (p^{(2)} - \\overline{p})^2. The F-statistic proposed by Alpaydin (see paper for justifications) is then computed as \\mathcal{f} = \\frac{\\sum_{i=1}^{5} \\sum_{j=1}^2 (p_i^{j})^2}{2 \\sum_{i=1}^5 s_i^2}, which is approximately F distributed with 10 and 5 degress of freedom. Using the f statistic, the p value can be computed and compared with a previously chosen significance level, e.g., \\alpha=0.05 . If the p value is smaller than \\alpha , we reject the null hypothesis and accept that there is a significant difference in the two models.","title":"Overview"},{"location":"user_guide/evaluate/combined_ftest_5x2cv/#references","text":"[1] Alpaydin, E. (1999). Combined 5\u00d72 cv F test for comparing supervised classification learning algorithms. Neural computation, 11(8), 1885-1892. [2] Dietterich TG (1998) Approximate Statistical Tests for Comparing Supervised Classification Learning Algorithms. Neural Comput 10:1895\u20131923.","title":"References"},{"location":"user_guide/evaluate/combined_ftest_5x2cv/#example-1-5x2cv-combined-f-test","text":"Assume we want to compare two classification algorithms, logistic regression and a decision tree algorithm: from sklearn.linear_model import LogisticRegression from sklearn.tree import DecisionTreeClassifier from mlxtend.data import iris_data from sklearn.model_selection import train_test_split X, y = iris_data() clf1 = LogisticRegression(random_state=1, solver='liblinear', multi_class='ovr') clf2 = DecisionTreeClassifier(random_state=1) X_train, X_test, y_train, y_test = \\ train_test_split(X, y, test_size=0.25, random_state=123) score1 = clf1.fit(X_train, y_train).score(X_test, y_test) score2 = clf2.fit(X_train, y_train).score(X_test, y_test) print('Logistic regression accuracy: %.2f%%' % (score1*100)) print('Decision tree accuracy: %.2f%%' % (score2*100)) Logistic regression accuracy: 97.37% Decision tree accuracy: 94.74% Note that these accuracy values are not used in the paired f test procedure as new test/train splits are generated during the resampling procedure, the values above are just serving the purpose of intuition. Now, let's assume a significance threshold of \\alpha=0.05 for rejecting the null hypothesis that both algorithms perform equally well on the dataset and conduct the 5x2cv f test: from mlxtend.evaluate import combined_ftest_5x2cv f, p = combined_ftest_5x2cv(estimator1=clf1, estimator2=clf2, X=X, y=y, random_seed=1) print('F statistic: %.3f' % f) print('p value: %.3f' % p) F statistic: 1.053 p value: 0.509 Since p > \\alpha , we cannot reject the null hypothesis and may conclude that the performance of the two algorithms is not significantly different. While it is generally not recommended to apply statistical tests multiple times without correction for multiple hypothesis testing, let us take a look at an example where the decision tree algorithm is limited to producing a very simple decision boundary that would result in a relatively bad performance: clf2 = DecisionTreeClassifier(random_state=1, max_depth=1) score2 = clf2.fit(X_train, y_train).score(X_test, y_test) print('Decision tree accuracy: %.2f%%' % (score2*100)) f, p = combined_ftest_5x2cv(estimator1=clf1, estimator2=clf2, X=X, y=y, random_seed=1) print('F statistic: %.3f' % f) print('p value: %.3f' % p) Decision tree accuracy: 63.16% F statistic: 34.934 p value: 0.001 Assuming that we conducted this test also with a significance level of \\alpha=0.05 , we can reject the null-hypothesis that both models perform equally well on this dataset, since the p-value ( p < 0.001 ) is smaller than \\alpha .","title":"Example 1 - 5x2cv combined F test"},{"location":"user_guide/evaluate/combined_ftest_5x2cv/#api","text":"combined_ftest_5x2cv(estimator1, estimator2, X, y, scoring=None, random_seed=None) Implements the 5x2cv combined F test proposed by Alpaydin 1999, to compare the performance of two models. Parameters estimator1 : scikit-learn classifier or regressor estimator2 : scikit-learn classifier or regressor X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values. scoring : str, callable, or None (default: None) If None (default), uses 'accuracy' for sklearn classifiers and 'r2' for sklearn regressors. If str, uses a sklearn scoring metric string identifier, for example {accuracy, f1, precision, recall, roc_auc} for classifiers, {'mean_absolute_error', 'mean_squared_error'/'neg_mean_squared_error', 'median_absolute_error', 'r2'} for regressors. If a callable object or function is provided, it has to be conform with sklearn's signature scorer(estimator, X, y) ; see https://scikit-learn.org/stable/modules/generated/sklearn.metrics.make_scorer.html for more information. random_seed : int or None (default: None) Random seed for creating the test/train splits. Returns f : float The F-statistic pvalue : float Two-tailed p-value. If the chosen significance level is larger than the p-value, we reject the null hypothesis and accept that there are significant differences in the two compared models. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/combined_ftest_5x2cv/","title":"API"},{"location":"user_guide/evaluate/confusion_matrix/","text":"confusion_matrix: creating a confusion matrix for model evaluation Functions for generating confusion matrices. from mlxtend.evaluate import confusion_matrix from mlxtend.plotting import plot_confusion_matrix Overview Confusion Matrix The confusion matrix (or error matrix ) is one way to summarize the performance of a classifier for binary classification tasks. This square matrix consists of columns and rows that list the number of instances as absolute or relative \"actual class\" vs. \"predicted class\" ratios. Let P be the label of class 1 and N be the label of a second class or the label of all classes that are not class 1 in a multi-class setting. References - Example 1 - Binary classification from mlxtend.evaluate import confusion_matrix y_target = [0, 0, 1, 0, 0, 1, 1, 1] y_predicted = [1, 0, 1, 0, 0, 0, 0, 1] cm = confusion_matrix(y_target=y_target, y_predicted=y_predicted) cm array([[3, 1], [2, 2]]) To visualize the confusion matrix using matplotlib, see the utility function mlxtend.plotting.plot_confusion_matrix : import matplotlib.pyplot as plt from mlxtend.plotting import plot_confusion_matrix fig, ax = plot_confusion_matrix(conf_mat=cm) plt.show() Example 2 - Multi-class classification from mlxtend.evaluate import confusion_matrix y_target = [1, 1, 1, 0, 0, 2, 0, 3] y_predicted = [1, 0, 1, 0, 0, 2, 1, 3] cm = confusion_matrix(y_target=y_target, y_predicted=y_predicted, binary=False) cm array([[2, 1, 0, 0], [1, 2, 0, 0], [0, 0, 1, 0], [0, 0, 0, 1]]) To visualize the confusion matrix using matplotlib, see the utility function mlxtend.plotting.plot_confusion_matrix : import matplotlib.pyplot as plt from mlxtend.evaluate import confusion_matrix fig, ax = plot_confusion_matrix(conf_mat=cm) plt.show() Example 3 - Multi-class to binary By setting binary=True , all class labels that are not the positive class label are being summarized to class 0. The positive class label becomes class 1. import matplotlib.pyplot as plt from mlxtend.evaluate import confusion_matrix y_target = [1, 1, 1, 0, 0, 2, 0, 3] y_predicted = [1, 0, 1, 0, 0, 2, 1, 3] cm = confusion_matrix(y_target=y_target, y_predicted=y_predicted, binary=True, positive_label=1) cm array([[4, 1], [1, 2]]) To visualize the confusion matrix using matplotlib, see the utility function mlxtend.plotting.plot_confusion_matrix : from mlxtend.plotting import plot_confusion_matrix fig, ax = plot_confusion_matrix(conf_mat=cm) plt.show() API confusion_matrix(y_target, y_predicted, binary=False, positive_label=1) Compute a confusion matrix/contingency table. Parameters y_target : array-like, shape=[n_samples] True class labels. y_predicted : array-like, shape=[n_samples] Predicted class labels. binary : bool (default: False) Maps a multi-class problem onto a binary confusion matrix, where the positive class is 1 and all other classes are 0. positive_label : int (default: 1) Class label of the positive class. Returns mat : array-like, shape=[n_classes, n_classes] Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/confusion_matrix/","title":"confusion_matrix: creating a confusion matrix for model evaluation"},{"location":"user_guide/evaluate/confusion_matrix/#confusion_matrix-creating-a-confusion-matrix-for-model-evaluation","text":"Functions for generating confusion matrices. from mlxtend.evaluate import confusion_matrix from mlxtend.plotting import plot_confusion_matrix","title":"confusion_matrix: creating a confusion matrix for model evaluation"},{"location":"user_guide/evaluate/confusion_matrix/#overview","text":"","title":"Overview"},{"location":"user_guide/evaluate/confusion_matrix/#confusion-matrix","text":"The confusion matrix (or error matrix ) is one way to summarize the performance of a classifier for binary classification tasks. This square matrix consists of columns and rows that list the number of instances as absolute or relative \"actual class\" vs. \"predicted class\" ratios. Let P be the label of class 1 and N be the label of a second class or the label of all classes that are not class 1 in a multi-class setting.","title":"Confusion Matrix"},{"location":"user_guide/evaluate/confusion_matrix/#references","text":"-","title":"References"},{"location":"user_guide/evaluate/confusion_matrix/#example-1-binary-classification","text":"from mlxtend.evaluate import confusion_matrix y_target = [0, 0, 1, 0, 0, 1, 1, 1] y_predicted = [1, 0, 1, 0, 0, 0, 0, 1] cm = confusion_matrix(y_target=y_target, y_predicted=y_predicted) cm array([[3, 1], [2, 2]]) To visualize the confusion matrix using matplotlib, see the utility function mlxtend.plotting.plot_confusion_matrix : import matplotlib.pyplot as plt from mlxtend.plotting import plot_confusion_matrix fig, ax = plot_confusion_matrix(conf_mat=cm) plt.show()","title":"Example 1 - Binary classification"},{"location":"user_guide/evaluate/confusion_matrix/#example-2-multi-class-classification","text":"from mlxtend.evaluate import confusion_matrix y_target = [1, 1, 1, 0, 0, 2, 0, 3] y_predicted = [1, 0, 1, 0, 0, 2, 1, 3] cm = confusion_matrix(y_target=y_target, y_predicted=y_predicted, binary=False) cm array([[2, 1, 0, 0], [1, 2, 0, 0], [0, 0, 1, 0], [0, 0, 0, 1]]) To visualize the confusion matrix using matplotlib, see the utility function mlxtend.plotting.plot_confusion_matrix : import matplotlib.pyplot as plt from mlxtend.evaluate import confusion_matrix fig, ax = plot_confusion_matrix(conf_mat=cm) plt.show()","title":"Example 2 - Multi-class classification"},{"location":"user_guide/evaluate/confusion_matrix/#example-3-multi-class-to-binary","text":"By setting binary=True , all class labels that are not the positive class label are being summarized to class 0. The positive class label becomes class 1. import matplotlib.pyplot as plt from mlxtend.evaluate import confusion_matrix y_target = [1, 1, 1, 0, 0, 2, 0, 3] y_predicted = [1, 0, 1, 0, 0, 2, 1, 3] cm = confusion_matrix(y_target=y_target, y_predicted=y_predicted, binary=True, positive_label=1) cm array([[4, 1], [1, 2]]) To visualize the confusion matrix using matplotlib, see the utility function mlxtend.plotting.plot_confusion_matrix : from mlxtend.plotting import plot_confusion_matrix fig, ax = plot_confusion_matrix(conf_mat=cm) plt.show()","title":"Example 3 - Multi-class to binary"},{"location":"user_guide/evaluate/confusion_matrix/#api","text":"confusion_matrix(y_target, y_predicted, binary=False, positive_label=1) Compute a confusion matrix/contingency table. Parameters y_target : array-like, shape=[n_samples] True class labels. y_predicted : array-like, shape=[n_samples] Predicted class labels. binary : bool (default: False) Maps a multi-class problem onto a binary confusion matrix, where the positive class is 1 and all other classes are 0. positive_label : int (default: 1) Class label of the positive class. Returns mat : array-like, shape=[n_classes, n_classes] Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/confusion_matrix/","title":"API"},{"location":"user_guide/evaluate/create_counterfactual/","text":"create_counterfactual: Interpreting models via counterfactuals An implementation of the counterfactual method by Wachter et al. 2017 for model interpretability. from mlxtend.evaluate import create_counterfactual Overview Counterfactuals are instances that explain scenarios related to implication: \"if not x, then not y\" in a hypothetical context. For example, \"if I hadn't studied hard, my grade would be worse.\" In the context of machine learning, we can think of counterfactual instances from the training set for which we artificially change its features to change the model prediction. Changing features of a training example can be useful to interpret the behavior of the model. Note that this implementation for creating counterfactuals is model agnostic and works with any scikit-learn estimators that support the predict (and ideally predict_proba ) method. In particular, the create_counterfactual implements the method described by Wachter et al. 2017 [1]. A good, short description of this method is also available in C. Molnar's Interpretable Machine Learning Book [2]. In short, Wachter et al.'s method minimizes the loss L\\left(x, x^{\\prime}, y^{\\prime}, \\lambda\\right)=\\lambda \\cdot\\left(\\hat{f}\\left(x^{\\prime}\\right)-y^{\\prime}\\right)^{2}+d\\left(x, x^{\\prime}\\right). The left term, \\lambda \\cdot\\left(\\hat{f}\\left(x^{\\prime}\\right)-y^{\\prime}\\right)^{2} , minimizes the squared difference between the model prediction for the counterfactual x' , i.e., \\hat{f}\\left(x^{\\prime}\\right) , and the desired prediction (specified by the user), y^{\\prime} . Note that \\lambda is a hyperparameter for weighting the importance of this left term over the second term, d\\left(x, x^{\\prime}\\right) . The second term, d\\left(x, x^{\\prime}\\right) , calculates the distance between a given instance x and a generated counterfactual x' . In short, the second term will keep the generated counterfactual similar to the instance. In contrast, the first term maximizes the difference between the model prediction for the counterfactual and the desired prediction (for example, a different class label). The distance function is implemented as the absolute difference in each feature dimension scaled by the median absolute deviation (MAD): d\\left(x, x^{\\prime}\\right)=\\sum_{j=1}^{p} \\frac{\\left|x_{j}-x_{j}^{\\prime}\\right|}{M A D_{j}}. The MAD measures the spread of a given feature, using the median as its center: MAD_{j}=\\operatorname{median}_{i \\in\\{1, \\ldots, n\\}}\\left(\\left|x_{i, j}-\\operatorname{median}_{l \\in\\{1, \\ldots, n\\}}\\left(x_{l, j}\\right)\\right|\\right). The general procedure for using the create_counterfactual function is as follows. Select an instance that you want to explain and specify the desired prediction y' for this instance (this is usually different from its original prediction). Choose a value for the hyperparameter \\lambda Optimize the loss L using the create_counterfactual function Optionally, as the authors recommend, you can repeat steps 2 and 3 by increasing \\lambda until a user-defined threshold \\epsilon is reached, i.e., while \\left|\\hat{f}\\left(x^{\\prime}\\right)-y^{\\prime}\\right|>\\epsilon : increase \\lambda References [1] Wachter, S., Mittelstadt, B., & Russell, C. (2017). Counterfactual explanations without opening the black box: Automated decisions and the GDPR. Harv. JL & Tech., 31, 841., https://arxiv.org/abs/1711.00399 [2] Christoph Molnar (2018). Interpretable Machine Learning, Chapter 6.1 Example 1 -- Simple Iris Example For simplicity, this example illustrates how to use the create_counterfactual function to explain a data instance from the iris dataset. Suppose we trained a logistic regression model on the iris dataset and pick the 16th training point for which we want to explain the prediction via counterfactuals. from mlxtend.data import iris_data from sklearn.linear_model import LogisticRegression X, y = iris_data() clf = LogisticRegression() clf.fit(X, y) x_ref = X[15] print('True label:', y[15]) print('Predicted label:', clf.predict(x_ref.reshape(1, -1))[0]) print('Predicted probas:', clf.predict_proba(x_ref.reshape(1, -1))) print('Predicted probability for label 0:', clf.predict_proba(x_ref.reshape(1, -1))[0][0]) True label: 0 Predicted label: 0 Predicted probas: [[9.86677291e-01 1.33226960e-02 1.28980184e-08]] Predicted probability for label 0: 0.9866772910539873 We can see above, that there is a predicted score of 98.6% probability for a class 0 membership. Now, we are going to push the prediction towards class 2 by setting y_desired=2 . Moreover, we set the probability for class 2 to 100% via y_desired_proba=1. from mlxtend.evaluate import create_counterfactual res = create_counterfactual(x_reference=x_ref, y_desired=2, model=clf, X_dataset=X, y_desired_proba=1., lammbda=1, # hyperparameter random_seed=123) print('Features of the 16th training example:', x_ref) print('Features of the countefactual:', res) print('Predictions for counterfactual:\\n') print('Predicted label:', clf.predict(res.reshape(1, -1))[0]) print('Predicted probas:', clf.predict_proba(res.reshape(1, -1))) Features of the 16th training example: [5.7 4.4 1.5 0.4] Features of the countefactual: [5.72271344 3.99169005 6.45305374 0.40000002] Predictions for counterfactual: Predicted label: 2 Predicted probas: [[1.41639932e-04 3.13292297e-01 6.86566063e-01]] As we can see above, the counterfactual is relatively similar to the original training example, i.e, only the 3rd feature has changed substantially (from 1.5 to 6.45). The predicted label has changed from class 0 t class 2. Interpretation-wise, this means increasing the petal length of a Iris-setosa flower may make it more similar to a Iris-virginica flower. Example 2 -- Simple Iris Example with Decision Regions and Threshold Stopping Criterion This example is similar to Example 1; however, it is based on a 2D iris dataset containing only petal length and petal width features so that the results can be plotted via a decision region plot. from mlxtend.plotting import plot_decision_regions import matplotlib.pyplot as plt X, y = iris_data() X = X[:, 2:] clf = LogisticRegression() clf.fit(X, y) LogisticRegression() # Plotting decision regions ax = plot_decision_regions(X, y, clf=clf, legend=2) scatter_highlight_defaults = {'c': 'red', 'edgecolor': 'yellow', 'alpha': 1.0, 'linewidths': 2, 'marker': 'o', 's': 80} ax.scatter(*X[15], **scatter_highlight_defaults) plt.show() The big, highlighted point in the plot above shows the 16th training datapoint. The following code will create a counterfactual with the same settings as in Example 1: counterfact = create_counterfactual(x_reference=X[15], y_desired=2, model=clf, X_dataset=X, y_desired_proba=1.0, lammbda=1, random_seed=123) ax = plot_decision_regions(X, y, clf=clf, legend=2) ax.scatter(*counterfact, **scatter_highlight_defaults) plt.show() As we can see above, the counterfactual primarily moved along the x-axis (petal length) so that the prediction between the rerence point and the counterfactual changes from class 0 to class 2. The following plots are based on repeating this procedure with different lambda values: for i in [0.4, 0.5, 1.0, 5.0, 100]: counterfact = create_counterfactual(x_reference=X[15], y_desired=2, model=clf, X_dataset=X, y_desired_proba=1.0, lammbda=i, random_seed=123) ax = plot_decision_regions(X, y, clf=clf, legend=2) ax.scatter(*counterfact, **scatter_highlight_defaults) plt.show() As we can see, the stronger the \\lambda value, the more the first term in the loss L\\left(x, x^{\\prime}, y^{\\prime}, \\lambda\\right)=\\lambda \\cdot\\left(\\hat{f}\\left(x^{\\prime}\\right)-y^{\\prime}\\right)^{2}+d\\left(x, x^{\\prime}\\right). dominates. Applying Wachter et al.'s threshold concept, Optionally, as the authors recommend, you can repeat steps 2 and 3 by increasing \\lambda until a user-defined threshold \\epsilon is reached, i.e., while \\left|\\hat{f}\\left(x^{\\prime}\\right)-y^{\\prime}\\right|>\\epsilon : increase \\lambda we can define a user-defined threshold and implement it as follows: import numpy as np desired_class_2_proba = 1.0 for i in np.arange(0, 10000, 0.1): counterfact = create_counterfactual(x_reference=X[15], y_desired=2, model=clf, X_dataset=X, y_desired_proba=desired_class_2_proba, lammbda=i, random_seed=123) predicted_class_2_proba = clf.predict_proba(counterfact.reshape(1, -1))[0][2] if not i: print('Initial lambda:', i) print('Initial diff:', np.abs(predicted_class_2_proba - desired_class_2_proba)) if not np.abs(predicted_class_2_proba - desired_class_2_proba) > 0.3: break ax = plot_decision_regions(X, y, clf=clf, legend=2) ax.scatter(*counterfact, **scatter_highlight_defaults) print('Final lambda:', i) print('Final diff:', np.abs(predicted_class_2_proba - desired_class_2_proba)) plt.show() Initial lambda: 0.0 Initial diff: 0.9999998976132334 Final lambda: 1.1 Final diff: 0.2962621523225484 API create_counterfactual(x_reference, y_desired, model, X_dataset, y_desired_proba=None, lammbda=0.1, random_seed=None) Implementation of the counterfactual method by Wachter et al. References: Wachter, S., Mittelstadt, B., & Russell, C. (2017). Counterfactual explanations without opening the black box: Automated decisions and the GDPR. Harv. JL & Tech., 31, 841., https://arxiv.org/abs/1711.00399 Parameters x_reference : array-like, shape=[m_features] The data instance (training example) to be explained. y_desired : int The desired class label for x_reference . model : estimator A (scikit-learn) estimator implementing .predict() and/or predict_proba() . - If model supports predict_proba() , then this is used by default for the first loss term, (lambda * model.predict[_proba](x_counterfact) - y_desired[_proba])^2 - Otherwise, method will fall back to predict . X_dataset : array-like, shape=[n_examples, m_features] A (training) dataset for picking the initial counterfactual as initial value for starting the optimization procedure. y_desired_proba : float (default: None) A float within the range [0, 1] designating the desired class probability for y_desired . - If y_desired_proba=None (default), the first loss term is (lambda * model(x_counterfact) - y_desired)^2 where y_desired is a class label - If y_desired_proba is not None, the first loss term is (lambda * model(x_counterfact) - y_desired_proba)^2 lammbda : Weighting parameter for the first loss term, (lambda * model(x_counterfact) - y_desired[_proba])^2 random_seed : int (default=None) If int, random_seed is the seed used by the random number generator for selecting the inital counterfactual from X_dataset . ython","title":"create_counterfactual: Interpreting models via counterfactuals"},{"location":"user_guide/evaluate/create_counterfactual/#create_counterfactual-interpreting-models-via-counterfactuals","text":"An implementation of the counterfactual method by Wachter et al. 2017 for model interpretability. from mlxtend.evaluate import create_counterfactual","title":"create_counterfactual: Interpreting models via counterfactuals"},{"location":"user_guide/evaluate/create_counterfactual/#overview","text":"Counterfactuals are instances that explain scenarios related to implication: \"if not x, then not y\" in a hypothetical context. For example, \"if I hadn't studied hard, my grade would be worse.\" In the context of machine learning, we can think of counterfactual instances from the training set for which we artificially change its features to change the model prediction. Changing features of a training example can be useful to interpret the behavior of the model. Note that this implementation for creating counterfactuals is model agnostic and works with any scikit-learn estimators that support the predict (and ideally predict_proba ) method. In particular, the create_counterfactual implements the method described by Wachter et al. 2017 [1]. A good, short description of this method is also available in C. Molnar's Interpretable Machine Learning Book [2]. In short, Wachter et al.'s method minimizes the loss L\\left(x, x^{\\prime}, y^{\\prime}, \\lambda\\right)=\\lambda \\cdot\\left(\\hat{f}\\left(x^{\\prime}\\right)-y^{\\prime}\\right)^{2}+d\\left(x, x^{\\prime}\\right). The left term, \\lambda \\cdot\\left(\\hat{f}\\left(x^{\\prime}\\right)-y^{\\prime}\\right)^{2} , minimizes the squared difference between the model prediction for the counterfactual x' , i.e., \\hat{f}\\left(x^{\\prime}\\right) , and the desired prediction (specified by the user), y^{\\prime} . Note that \\lambda is a hyperparameter for weighting the importance of this left term over the second term, d\\left(x, x^{\\prime}\\right) . The second term, d\\left(x, x^{\\prime}\\right) , calculates the distance between a given instance x and a generated counterfactual x' . In short, the second term will keep the generated counterfactual similar to the instance. In contrast, the first term maximizes the difference between the model prediction for the counterfactual and the desired prediction (for example, a different class label). The distance function is implemented as the absolute difference in each feature dimension scaled by the median absolute deviation (MAD): d\\left(x, x^{\\prime}\\right)=\\sum_{j=1}^{p} \\frac{\\left|x_{j}-x_{j}^{\\prime}\\right|}{M A D_{j}}. The MAD measures the spread of a given feature, using the median as its center: MAD_{j}=\\operatorname{median}_{i \\in\\{1, \\ldots, n\\}}\\left(\\left|x_{i, j}-\\operatorname{median}_{l \\in\\{1, \\ldots, n\\}}\\left(x_{l, j}\\right)\\right|\\right). The general procedure for using the create_counterfactual function is as follows. Select an instance that you want to explain and specify the desired prediction y' for this instance (this is usually different from its original prediction). Choose a value for the hyperparameter \\lambda Optimize the loss L using the create_counterfactual function Optionally, as the authors recommend, you can repeat steps 2 and 3 by increasing \\lambda until a user-defined threshold \\epsilon is reached, i.e., while \\left|\\hat{f}\\left(x^{\\prime}\\right)-y^{\\prime}\\right|>\\epsilon : increase \\lambda","title":"Overview"},{"location":"user_guide/evaluate/create_counterfactual/#references","text":"[1] Wachter, S., Mittelstadt, B., & Russell, C. (2017). Counterfactual explanations without opening the black box: Automated decisions and the GDPR. Harv. JL & Tech., 31, 841., https://arxiv.org/abs/1711.00399 [2] Christoph Molnar (2018). Interpretable Machine Learning, Chapter 6.1","title":"References"},{"location":"user_guide/evaluate/create_counterfactual/#example-1-simple-iris-example","text":"For simplicity, this example illustrates how to use the create_counterfactual function to explain a data instance from the iris dataset. Suppose we trained a logistic regression model on the iris dataset and pick the 16th training point for which we want to explain the prediction via counterfactuals. from mlxtend.data import iris_data from sklearn.linear_model import LogisticRegression X, y = iris_data() clf = LogisticRegression() clf.fit(X, y) x_ref = X[15] print('True label:', y[15]) print('Predicted label:', clf.predict(x_ref.reshape(1, -1))[0]) print('Predicted probas:', clf.predict_proba(x_ref.reshape(1, -1))) print('Predicted probability for label 0:', clf.predict_proba(x_ref.reshape(1, -1))[0][0]) True label: 0 Predicted label: 0 Predicted probas: [[9.86677291e-01 1.33226960e-02 1.28980184e-08]] Predicted probability for label 0: 0.9866772910539873 We can see above, that there is a predicted score of 98.6% probability for a class 0 membership. Now, we are going to push the prediction towards class 2 by setting y_desired=2 . Moreover, we set the probability for class 2 to 100% via y_desired_proba=1. from mlxtend.evaluate import create_counterfactual res = create_counterfactual(x_reference=x_ref, y_desired=2, model=clf, X_dataset=X, y_desired_proba=1., lammbda=1, # hyperparameter random_seed=123) print('Features of the 16th training example:', x_ref) print('Features of the countefactual:', res) print('Predictions for counterfactual:\\n') print('Predicted label:', clf.predict(res.reshape(1, -1))[0]) print('Predicted probas:', clf.predict_proba(res.reshape(1, -1))) Features of the 16th training example: [5.7 4.4 1.5 0.4] Features of the countefactual: [5.72271344 3.99169005 6.45305374 0.40000002] Predictions for counterfactual: Predicted label: 2 Predicted probas: [[1.41639932e-04 3.13292297e-01 6.86566063e-01]] As we can see above, the counterfactual is relatively similar to the original training example, i.e, only the 3rd feature has changed substantially (from 1.5 to 6.45). The predicted label has changed from class 0 t class 2. Interpretation-wise, this means increasing the petal length of a Iris-setosa flower may make it more similar to a Iris-virginica flower.","title":"Example 1 -- Simple Iris Example"},{"location":"user_guide/evaluate/create_counterfactual/#example-2-simple-iris-example-with-decision-regions-and-threshold-stopping-criterion","text":"This example is similar to Example 1; however, it is based on a 2D iris dataset containing only petal length and petal width features so that the results can be plotted via a decision region plot. from mlxtend.plotting import plot_decision_regions import matplotlib.pyplot as plt X, y = iris_data() X = X[:, 2:] clf = LogisticRegression() clf.fit(X, y) LogisticRegression() # Plotting decision regions ax = plot_decision_regions(X, y, clf=clf, legend=2) scatter_highlight_defaults = {'c': 'red', 'edgecolor': 'yellow', 'alpha': 1.0, 'linewidths': 2, 'marker': 'o', 's': 80} ax.scatter(*X[15], **scatter_highlight_defaults) plt.show() The big, highlighted point in the plot above shows the 16th training datapoint. The following code will create a counterfactual with the same settings as in Example 1: counterfact = create_counterfactual(x_reference=X[15], y_desired=2, model=clf, X_dataset=X, y_desired_proba=1.0, lammbda=1, random_seed=123) ax = plot_decision_regions(X, y, clf=clf, legend=2) ax.scatter(*counterfact, **scatter_highlight_defaults) plt.show() As we can see above, the counterfactual primarily moved along the x-axis (petal length) so that the prediction between the rerence point and the counterfactual changes from class 0 to class 2. The following plots are based on repeating this procedure with different lambda values: for i in [0.4, 0.5, 1.0, 5.0, 100]: counterfact = create_counterfactual(x_reference=X[15], y_desired=2, model=clf, X_dataset=X, y_desired_proba=1.0, lammbda=i, random_seed=123) ax = plot_decision_regions(X, y, clf=clf, legend=2) ax.scatter(*counterfact, **scatter_highlight_defaults) plt.show() As we can see, the stronger the \\lambda value, the more the first term in the loss L\\left(x, x^{\\prime}, y^{\\prime}, \\lambda\\right)=\\lambda \\cdot\\left(\\hat{f}\\left(x^{\\prime}\\right)-y^{\\prime}\\right)^{2}+d\\left(x, x^{\\prime}\\right). dominates. Applying Wachter et al.'s threshold concept, Optionally, as the authors recommend, you can repeat steps 2 and 3 by increasing \\lambda until a user-defined threshold \\epsilon is reached, i.e., while \\left|\\hat{f}\\left(x^{\\prime}\\right)-y^{\\prime}\\right|>\\epsilon : increase \\lambda we can define a user-defined threshold and implement it as follows: import numpy as np desired_class_2_proba = 1.0 for i in np.arange(0, 10000, 0.1): counterfact = create_counterfactual(x_reference=X[15], y_desired=2, model=clf, X_dataset=X, y_desired_proba=desired_class_2_proba, lammbda=i, random_seed=123) predicted_class_2_proba = clf.predict_proba(counterfact.reshape(1, -1))[0][2] if not i: print('Initial lambda:', i) print('Initial diff:', np.abs(predicted_class_2_proba - desired_class_2_proba)) if not np.abs(predicted_class_2_proba - desired_class_2_proba) > 0.3: break ax = plot_decision_regions(X, y, clf=clf, legend=2) ax.scatter(*counterfact, **scatter_highlight_defaults) print('Final lambda:', i) print('Final diff:', np.abs(predicted_class_2_proba - desired_class_2_proba)) plt.show() Initial lambda: 0.0 Initial diff: 0.9999998976132334 Final lambda: 1.1 Final diff: 0.2962621523225484","title":"Example 2 -- Simple Iris Example with Decision Regions and Threshold Stopping Criterion"},{"location":"user_guide/evaluate/create_counterfactual/#api","text":"create_counterfactual(x_reference, y_desired, model, X_dataset, y_desired_proba=None, lammbda=0.1, random_seed=None) Implementation of the counterfactual method by Wachter et al. References: Wachter, S., Mittelstadt, B., & Russell, C. (2017). Counterfactual explanations without opening the black box: Automated decisions and the GDPR. Harv. JL & Tech., 31, 841., https://arxiv.org/abs/1711.00399 Parameters x_reference : array-like, shape=[m_features] The data instance (training example) to be explained. y_desired : int The desired class label for x_reference . model : estimator A (scikit-learn) estimator implementing .predict() and/or predict_proba() . - If model supports predict_proba() , then this is used by default for the first loss term, (lambda * model.predict[_proba](x_counterfact) - y_desired[_proba])^2 - Otherwise, method will fall back to predict . X_dataset : array-like, shape=[n_examples, m_features] A (training) dataset for picking the initial counterfactual as initial value for starting the optimization procedure. y_desired_proba : float (default: None) A float within the range [0, 1] designating the desired class probability for y_desired . - If y_desired_proba=None (default), the first loss term is (lambda * model(x_counterfact) - y_desired)^2 where y_desired is a class label - If y_desired_proba is not None, the first loss term is (lambda * model(x_counterfact) - y_desired_proba)^2 lammbda : Weighting parameter for the first loss term, (lambda * model(x_counterfact) - y_desired[_proba])^2 random_seed : int (default=None) If int, random_seed is the seed used by the random number generator for selecting the inital counterfactual from X_dataset . ython","title":"API"},{"location":"user_guide/evaluate/feature_importance_permutation/","text":"feature_importance_permutation: Estimate feature importance via feature permutation. A function to estimate the feature importance of classifiers and regressors based on permutation importance . from mlxtend.evaluate import feature_importance_permutation Overview The permutation importance is an intuitive, model-agnostic method to estimate the feature importance for classifier and regression models. The approach is relatively simple and straight-forward: Take a model that was fit to the training dataset Estimate the predictive performance of the model on an independent dataset (e.g., validation dataset) and record it as the baseline performance For each feature i : randomly permute feature column i in the original dataset record the predictive performance of the model on the dataset with the permuted column compute the feature importance as the difference between the baseline performance (step 2) and the performance on the permuted dataset Permutation importance is generally considered as a relatively efficient technique that works well in practice [1], while a drawback is that the importance of correlated features may be overestimated [2]. References [1] Terence Parr, Kerem Turgutlu, Christopher Csiszar, and Jeremy Howard. Beware Default Random Forest Importances (https://parrt.cs.usfca.edu/doc/rf-importance/index.html) [2] Strobl, C., Boulesteix, A. L., Kneib, T., Augustin, T., & Zeileis, A. (2008). Conditional variable importance for random forests. BMC bioinformatics, 9(1), 307. Example 1 -- Feature Importance for Classifiers The following example illustrates the feature importance estimation via permutation importance based for classification models. import numpy as np import matplotlib.pyplot as plt from sklearn.svm import SVC from sklearn.model_selection import train_test_split from mlxtend.evaluate import feature_importance_permutation Generate a toy dataset from sklearn.datasets import make_classification from sklearn.ensemble import RandomForestClassifier # Build a classification task using 3 informative features X, y = make_classification(n_samples=10000, n_features=10, n_informative=3, n_redundant=0, n_repeated=0, n_classes=2, random_state=0, shuffle=False) X_train, X_test, y_train, y_test = train_test_split( X, y, test_size=0.3, random_state=1, stratify=y) Feature importance via random forest First, we compute the feature importance directly from the random forest via mean impurity decrease (described after the code section): forest = RandomForestClassifier(n_estimators=250, random_state=0) forest.fit(X_train, y_train) print('Training accuracy:', np.mean(forest.predict(X_train) == y_train)*100) print('Test accuracy:', np.mean(forest.predict(X_test) == y_test)*100) importance_vals = forest.feature_importances_ print(importance_vals) Training accuracy: 100.0 Test accuracy: 95.06666666666666 [0.283357 0.30846795 0.24204291 0.02229767 0.02364941 0.02390578 0.02501543 0.0234225 0.02370816 0.0241332 ] There are several strategies for computing the feature importance in random forest. The method implemented in scikit-learn (used in the next code example) is based on the Breiman and Friedman's CART (Breiman, Friedman, \"Classification and regression trees\", 1984), the so-called mean impurity decrease . Here, the importance value of a features is computed by averaging the impurity decrease for that feature, when splitting a parent node into two child nodes, across all the trees in the ensemble. Note that the impurity decrease values are weighted by the number of samples that are in the respective nodes. This process is repeated for all features in the dataset, and the feature importance values are then normalized so that they sum up to 1. In CART, the authors also note that this fast way of computing feature importance values is relatively consistent with the permutation importance. Next, let's visualize the feature importance values from the random forest including a measure of the mean impurity decrease variability (here: standard deviation): std = np.std([tree.feature_importances_ for tree in forest.estimators_], axis=0) indices = np.argsort(importance_vals)[::-1] # Plot the feature importances of the forest plt.figure() plt.title(\"Random Forest feature importance\") plt.bar(range(X.shape[1]), importance_vals[indices], yerr=std[indices], align=\"center\") plt.xticks(range(X.shape[1]), indices) plt.xlim([-1, X.shape[1]]) plt.ylim([0, 0.5]) plt.show() As we can see, the features 1, 0, and 2 are estimated to be the most informative ones for the random forest classier. Next, let's compute the feature importance via the permutation importance approach. Permutation Importance imp_vals, _ = feature_importance_permutation( predict_method=forest.predict, X=X_test, y=y_test, metric='accuracy', num_rounds=1, seed=1) imp_vals array([ 0.26833333, 0.26733333, 0.261 , -0.002 , -0.00033333, 0.00066667, 0.00233333, 0.00066667, 0.00066667, -0.00233333]) Note that the feature_importance_permutation returns two arrays. The first array (here: imp_vals ) contains the actual importance values we are interested in. If num_rounds > 1 , the permutation is repeated multiple times (with different random seeds), and in this case the first array contains the average value of the importance computed from the different runs. The second array (here, assigned to _ , because we are not using it) then contains all individual values from these runs (more about that later). Now, let's also visualize the importance values in a barplot: indices = np.argsort(imp_vals)[::-1] plt.figure() plt.title(\"Random Forest feature importance via permutation importance\") plt.bar(range(X.shape[1]), imp_vals[indices]) plt.xticks(range(X.shape[1]), indices) plt.xlim([-1, X.shape[1]]) plt.ylim([0, 0.5]) plt.show() As we can see, also here, features 1, 0, and 2 are predicted to be the most important ones, which is consistent with the feature importance values that we computed via the mean impurity decrease method earlier. (Note that in the context of random forests, the feature importance via permutation importance is typically computed using the out-of-bag samples of a random forest, whereas in this implementation, an independent dataset is used.) Previously, it was mentioned that the permutation is repeated multiple times if num_rounds > 1 . In this case, the second array returned by the feature_importance_permutation contains the importance values for these individual runs (the array has shape [num_features, num_rounds), which we can use to compute some sort of variability between these runs. imp_vals, imp_all = feature_importance_permutation( predict_method=forest.predict, X=X_test, y=y_test, metric='accuracy', num_rounds=10, seed=1) std = np.std(imp_all, axis=1) indices = np.argsort(imp_vals)[::-1] plt.figure() plt.title(\"Random Forest feature importance via permutation importance w. std. dev.\") plt.bar(range(X.shape[1]), imp_vals[indices], yerr=std[indices]) plt.xticks(range(X.shape[1]), indices) plt.xlim([-1, X.shape[1]]) plt.show() It shall be noted that the feature importance values do not sum up to one, since they are not normalized (you can normalize them if you'd like, by dividing these by the sum of importance values). Here, the main point is to look at the importance values relative to each other and not to over-interpret the absolute values. Support Vector Machines While the permutation importance approach yields results that are generally consistent with the mean impurity decrease feature importance values from a random forest, it's a method that is model-agnostic and can be used with any kind of classifier or regressor. The example below applies the feature_importance_permutation function to a support vector machine: from sklearn.svm import SVC svm = SVC(C=1.0, kernel='rbf') svm.fit(X_train, y_train) print('Training accuracy', np.mean(svm.predict(X_train) == y_train)*100) print('Test accuracy', np.mean(svm.predict(X_test) == y_test)*100) Training accuracy 94.87142857142857 Test accuracy 94.89999999999999 imp_vals, imp_all = feature_importance_permutation( predict_method=svm.predict, X=X_test, y=y_test, metric='accuracy', num_rounds=10, seed=1) std = np.std(imp_all, axis=1) indices = np.argsort(imp_vals)[::-1] plt.figure() plt.title(\"SVM feature importance via permutation importance\") plt.bar(range(X.shape[1]), imp_vals[indices], yerr=std[indices]) plt.xticks(range(X.shape[1]), indices) plt.xlim([-1, X.shape[1]]) plt.show() Example 2 -- Feature Importance for Regressors import numpy as np import matplotlib.pyplot as plt from mlxtend.evaluate import feature_importance_permutation from sklearn.model_selection import train_test_split from sklearn.datasets import make_regression from sklearn.svm import SVR X, y = make_regression(n_samples=1000, n_features=5, n_informative=2, n_targets=1, random_state=123, shuffle=False) X_train, X_test, y_train, y_test = train_test_split( X, y, test_size=0.3, random_state=123) svm = SVR(kernel='rbf') svm.fit(X_train, y_train) imp_vals, _ = feature_importance_permutation( predict_method=svm.predict, X=X_test, y=y_test, metric='r2', num_rounds=1, seed=1) imp_vals array([ 0.43309137, 0.22058866, 0.00148447, 0.01613953, -0.00529505]) plt.figure() plt.bar(range(X.shape[1]), imp_vals) plt.xticks(range(X.shape[1])) plt.xlim([-1, X.shape[1]]) plt.ylim([0, 0.5]) plt.show() Example 3 -- Feature Importance With One-Hot-Encoded Features Upon one-hot encoding, a feature variable with 10 distinct categories will be split into 10 new feature columns (or 9, if you drop a redundant column). If we want to treat each new feature column as an individual feature variable, we can use the feature_importance_permutation as usual. This is illustrated in the example below. Preparing the Dataset Here, we look at a dataset that consists of one categorical feature ( 'categorical' ) and 3 numerical features ( 'measurement1' , 'measurement2' , and 'measurement3' ). import pandas as pd df_data = pd.read_csv('https://gist.githubusercontent.com/rasbt/b99bf69079bc0d601eeae8a49248d358/raw/a114be9801647ec5460089f3a9576713dabf5f1f/onehot-numeric-mixed-data.csv') df_data.head() .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } categorical measurement1 measurement2 label measurement3 0 F 1.428571 2.721313 0 2.089 1 R 0.685939 0.982976 0 0.637 2 P 1.055817 0.624210 0 0.226 3 S 0.995956 0.321101 0 0.138 4 R 1.376773 1.578309 0 0.478 from sklearn.model_selection import train_test_split df_X = df_data[['measurement1', 'measurement2', 'measurement3', 'categorical']] df_y = df_data['label'] df_X_train, df_X_test, df_y_train, df_y_test = train_test_split( df_X, df_y, test_size=0.33, random_state=42, stratify=df_y) Here, we do the one-hot encoding on the categorical feature and merge it back with the numerical columns: from sklearn.preprocessing import OneHotEncoder import numpy as np ohe = OneHotEncoder(drop='first') ohe.fit(df_X_train[['categorical']]) df_X_train_ohe = df_X_train.drop(columns=['categorical']) df_X_test_ohe = df_X_test.drop(columns=['categorical']) ohe_train = np.asarray(ohe.transform(df_X_train[['categorical']]).todense()) ohe_test = np.asarray(ohe.transform(df_X_test[['categorical']]).todense()) X_train_ohe = np.hstack((df_X_train_ohe.values, ohe_train)) X_test_ohe = np.hstack((df_X_test_ohe.values, ohe_test)) # look at first 3 rows print(X_train_ohe[:3]) [[0.65747208 0.95105388 0.36 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 1. 0. 0. 0. 0. 0. ] [1.17503636 1.01094494 0.653 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 1. 0. 0. 0. 0. ] [1.25516647 0.67575824 0.176 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 1. 0. 0. 0. 0. 0. 0. ]] Fitting a Baseline Model for Analysis from sklearn.neural_network import MLPClassifier from sklearn.preprocessing import StandardScaler from sklearn.pipeline import make_pipeline from sklearn.model_selection import GridSearchCV pipe = make_pipeline(StandardScaler(), MLPClassifier(max_iter=10000, random_state=123)) params = { 'mlpclassifier__hidden_layer_sizes': [(30, 20, 10), (20, 10), (20,), (10,)], 'mlpclassifier__activation': ['tanh', 'relu'], 'mlpclassifier__solver': ['sgd'], 'mlpclassifier__alpha': [0.0001], 'mlpclassifier__learning_rate': ['adaptive'], } gs = GridSearchCV(estimator=pipe, param_grid=params, scoring='accuracy', refit=True, n_jobs=-1, cv=10) gs = gs.fit(X_train_ohe, df_y_train.values) model = gs.best_estimator_ Regular Permutation Importance Here, we compute the feature importance the regular way were each one-hot encoded feature is treated as an individiual variable. from mlxtend.evaluate import feature_importance_permutation imp_vals, imp_all = feature_importance_permutation( predict_method=model.predict, X=X_test_ohe, y=df_y_test.values, metric='accuracy', num_rounds=50, seed=1) feat_names_with_ohe = ['measurement1', 'measurement2', 'measurement3'] \\ + [f'categorical_ohe_{i}' for i in range(2, 20)] %matplotlib inline import matplotlib.pyplot as plt std = np.std(imp_all, axis=1) indices = np.argsort(imp_vals)[::-1] plt.figure() #plt.title(\"Feature importance via permutation importance w. std. dev.\") plt.bar(range(len(feat_names_with_ohe)), imp_vals[indices], yerr=std[indices]) plt.xticks(range(len(feat_names_with_ohe)), np.array(feat_names_with_ohe)[indices], rotation=90) plt.xlim([-1, len(feat_names_with_ohe)]) plt.show() However, notice that if we have a lot of categorical values, the feature importance of the individual binary features (after one hot encoding) are now hard to interpret. In certain cases, it would be desireable to treat the one-hot encoded binary features as a single variable in feature permutation importance evaluation. We can achieve this using feature groups. With Feature Groups In the example below, all the one-hot encoded variables are treated as a feature group. This means, they are are all shuffled and analyzed as a single feature inside the feature permutation importance analysis. feature_groups = [0, 1, 2, range(3, 21)] imp_vals, imp_all = feature_importance_permutation( predict_method=model.predict, X=X_test_ohe, y=df_y_test.values, metric='accuracy', num_rounds=50, feature_groups=feature_groups, seed=1) feature_names = ['measurement1', 'measurement2', 'measurement3', 'categorical'] std = np.std(imp_all, axis=1) indices = np.argsort(imp_vals)[::-1] plt.figure() plt.bar(range(len(feature_names)), imp_vals[indices], yerr=std[indices]) plt.xticks(range(len(feature_names)), np.array(feature_names)[indices], rotation=90) plt.xlim([-1, len(feature_names)]) plt.show() API feature_importance_permutation(X, y, predict_method, metric, num_rounds=1, feature_groups=None, seed=None) Feature importance imputation via permutation importance Parameters X : NumPy array, shape = [n_samples, n_features] Dataset, where n_samples is the number of samples and n_features is the number of features. y : NumPy array, shape = [n_samples] Target values. predict_method : prediction function A callable function that predicts the target values from X. metric : str, callable The metric for evaluating the feature importance through permutation. By default, the strings 'accuracy' is recommended for classifiers and the string 'r2' is recommended for regressors. Optionally, a custom scoring function (e.g., metric=scoring_func ) that accepts two arguments, y_true and y_pred, which have similar shape to the y array. num_rounds : int (default=1) Number of rounds the feature columns are permuted to compute the permutation importance. feature_groups : list or None (default=None) Optional argument for treating certain features as a group. For example [1, 2, [3, 4, 5]] , which can be useful for interpretability, for example, if features 3, 4, 5 are one-hot encoded features. seed : int or None (default=None) Random seed for permuting the feature columns. Returns mean_importance_vals, all_importance_vals : NumPy arrays. The first array, mean_importance_vals has shape [n_features, ] and contains the importance values for all features. The shape of the second array is [n_features, num_rounds] and contains the feature importance for each repetition. If num_rounds=1, it contains the same values as the first array, mean_importance_vals. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/feature_importance_permutation/ ython ython","title":"feature_importance_permutation: Estimate feature importance via feature permutation."},{"location":"user_guide/evaluate/feature_importance_permutation/#feature_importance_permutation-estimate-feature-importance-via-feature-permutation","text":"A function to estimate the feature importance of classifiers and regressors based on permutation importance . from mlxtend.evaluate import feature_importance_permutation","title":"feature_importance_permutation: Estimate feature importance via feature permutation."},{"location":"user_guide/evaluate/feature_importance_permutation/#overview","text":"The permutation importance is an intuitive, model-agnostic method to estimate the feature importance for classifier and regression models. The approach is relatively simple and straight-forward: Take a model that was fit to the training dataset Estimate the predictive performance of the model on an independent dataset (e.g., validation dataset) and record it as the baseline performance For each feature i : randomly permute feature column i in the original dataset record the predictive performance of the model on the dataset with the permuted column compute the feature importance as the difference between the baseline performance (step 2) and the performance on the permuted dataset Permutation importance is generally considered as a relatively efficient technique that works well in practice [1], while a drawback is that the importance of correlated features may be overestimated [2].","title":"Overview"},{"location":"user_guide/evaluate/feature_importance_permutation/#references","text":"[1] Terence Parr, Kerem Turgutlu, Christopher Csiszar, and Jeremy Howard. Beware Default Random Forest Importances (https://parrt.cs.usfca.edu/doc/rf-importance/index.html) [2] Strobl, C., Boulesteix, A. L., Kneib, T., Augustin, T., & Zeileis, A. (2008). Conditional variable importance for random forests. BMC bioinformatics, 9(1), 307.","title":"References"},{"location":"user_guide/evaluate/feature_importance_permutation/#example-1-feature-importance-for-classifiers","text":"The following example illustrates the feature importance estimation via permutation importance based for classification models. import numpy as np import matplotlib.pyplot as plt from sklearn.svm import SVC from sklearn.model_selection import train_test_split from mlxtend.evaluate import feature_importance_permutation","title":"Example 1 -- Feature Importance for Classifiers"},{"location":"user_guide/evaluate/feature_importance_permutation/#generate-a-toy-dataset","text":"from sklearn.datasets import make_classification from sklearn.ensemble import RandomForestClassifier # Build a classification task using 3 informative features X, y = make_classification(n_samples=10000, n_features=10, n_informative=3, n_redundant=0, n_repeated=0, n_classes=2, random_state=0, shuffle=False) X_train, X_test, y_train, y_test = train_test_split( X, y, test_size=0.3, random_state=1, stratify=y)","title":"Generate a toy dataset"},{"location":"user_guide/evaluate/feature_importance_permutation/#feature-importance-via-random-forest","text":"First, we compute the feature importance directly from the random forest via mean impurity decrease (described after the code section): forest = RandomForestClassifier(n_estimators=250, random_state=0) forest.fit(X_train, y_train) print('Training accuracy:', np.mean(forest.predict(X_train) == y_train)*100) print('Test accuracy:', np.mean(forest.predict(X_test) == y_test)*100) importance_vals = forest.feature_importances_ print(importance_vals) Training accuracy: 100.0 Test accuracy: 95.06666666666666 [0.283357 0.30846795 0.24204291 0.02229767 0.02364941 0.02390578 0.02501543 0.0234225 0.02370816 0.0241332 ] There are several strategies for computing the feature importance in random forest. The method implemented in scikit-learn (used in the next code example) is based on the Breiman and Friedman's CART (Breiman, Friedman, \"Classification and regression trees\", 1984), the so-called mean impurity decrease . Here, the importance value of a features is computed by averaging the impurity decrease for that feature, when splitting a parent node into two child nodes, across all the trees in the ensemble. Note that the impurity decrease values are weighted by the number of samples that are in the respective nodes. This process is repeated for all features in the dataset, and the feature importance values are then normalized so that they sum up to 1. In CART, the authors also note that this fast way of computing feature importance values is relatively consistent with the permutation importance. Next, let's visualize the feature importance values from the random forest including a measure of the mean impurity decrease variability (here: standard deviation): std = np.std([tree.feature_importances_ for tree in forest.estimators_], axis=0) indices = np.argsort(importance_vals)[::-1] # Plot the feature importances of the forest plt.figure() plt.title(\"Random Forest feature importance\") plt.bar(range(X.shape[1]), importance_vals[indices], yerr=std[indices], align=\"center\") plt.xticks(range(X.shape[1]), indices) plt.xlim([-1, X.shape[1]]) plt.ylim([0, 0.5]) plt.show() As we can see, the features 1, 0, and 2 are estimated to be the most informative ones for the random forest classier. Next, let's compute the feature importance via the permutation importance approach.","title":"Feature importance via random forest"},{"location":"user_guide/evaluate/feature_importance_permutation/#permutation-importance","text":"imp_vals, _ = feature_importance_permutation( predict_method=forest.predict, X=X_test, y=y_test, metric='accuracy', num_rounds=1, seed=1) imp_vals array([ 0.26833333, 0.26733333, 0.261 , -0.002 , -0.00033333, 0.00066667, 0.00233333, 0.00066667, 0.00066667, -0.00233333]) Note that the feature_importance_permutation returns two arrays. The first array (here: imp_vals ) contains the actual importance values we are interested in. If num_rounds > 1 , the permutation is repeated multiple times (with different random seeds), and in this case the first array contains the average value of the importance computed from the different runs. The second array (here, assigned to _ , because we are not using it) then contains all individual values from these runs (more about that later). Now, let's also visualize the importance values in a barplot: indices = np.argsort(imp_vals)[::-1] plt.figure() plt.title(\"Random Forest feature importance via permutation importance\") plt.bar(range(X.shape[1]), imp_vals[indices]) plt.xticks(range(X.shape[1]), indices) plt.xlim([-1, X.shape[1]]) plt.ylim([0, 0.5]) plt.show() As we can see, also here, features 1, 0, and 2 are predicted to be the most important ones, which is consistent with the feature importance values that we computed via the mean impurity decrease method earlier. (Note that in the context of random forests, the feature importance via permutation importance is typically computed using the out-of-bag samples of a random forest, whereas in this implementation, an independent dataset is used.) Previously, it was mentioned that the permutation is repeated multiple times if num_rounds > 1 . In this case, the second array returned by the feature_importance_permutation contains the importance values for these individual runs (the array has shape [num_features, num_rounds), which we can use to compute some sort of variability between these runs. imp_vals, imp_all = feature_importance_permutation( predict_method=forest.predict, X=X_test, y=y_test, metric='accuracy', num_rounds=10, seed=1) std = np.std(imp_all, axis=1) indices = np.argsort(imp_vals)[::-1] plt.figure() plt.title(\"Random Forest feature importance via permutation importance w. std. dev.\") plt.bar(range(X.shape[1]), imp_vals[indices], yerr=std[indices]) plt.xticks(range(X.shape[1]), indices) plt.xlim([-1, X.shape[1]]) plt.show() It shall be noted that the feature importance values do not sum up to one, since they are not normalized (you can normalize them if you'd like, by dividing these by the sum of importance values). Here, the main point is to look at the importance values relative to each other and not to over-interpret the absolute values.","title":"Permutation Importance"},{"location":"user_guide/evaluate/feature_importance_permutation/#support-vector-machines","text":"While the permutation importance approach yields results that are generally consistent with the mean impurity decrease feature importance values from a random forest, it's a method that is model-agnostic and can be used with any kind of classifier or regressor. The example below applies the feature_importance_permutation function to a support vector machine: from sklearn.svm import SVC svm = SVC(C=1.0, kernel='rbf') svm.fit(X_train, y_train) print('Training accuracy', np.mean(svm.predict(X_train) == y_train)*100) print('Test accuracy', np.mean(svm.predict(X_test) == y_test)*100) Training accuracy 94.87142857142857 Test accuracy 94.89999999999999 imp_vals, imp_all = feature_importance_permutation( predict_method=svm.predict, X=X_test, y=y_test, metric='accuracy', num_rounds=10, seed=1) std = np.std(imp_all, axis=1) indices = np.argsort(imp_vals)[::-1] plt.figure() plt.title(\"SVM feature importance via permutation importance\") plt.bar(range(X.shape[1]), imp_vals[indices], yerr=std[indices]) plt.xticks(range(X.shape[1]), indices) plt.xlim([-1, X.shape[1]]) plt.show()","title":"Support Vector Machines"},{"location":"user_guide/evaluate/feature_importance_permutation/#example-2-feature-importance-for-regressors","text":"import numpy as np import matplotlib.pyplot as plt from mlxtend.evaluate import feature_importance_permutation from sklearn.model_selection import train_test_split from sklearn.datasets import make_regression from sklearn.svm import SVR X, y = make_regression(n_samples=1000, n_features=5, n_informative=2, n_targets=1, random_state=123, shuffle=False) X_train, X_test, y_train, y_test = train_test_split( X, y, test_size=0.3, random_state=123) svm = SVR(kernel='rbf') svm.fit(X_train, y_train) imp_vals, _ = feature_importance_permutation( predict_method=svm.predict, X=X_test, y=y_test, metric='r2', num_rounds=1, seed=1) imp_vals array([ 0.43309137, 0.22058866, 0.00148447, 0.01613953, -0.00529505]) plt.figure() plt.bar(range(X.shape[1]), imp_vals) plt.xticks(range(X.shape[1])) plt.xlim([-1, X.shape[1]]) plt.ylim([0, 0.5]) plt.show()","title":"Example 2 -- Feature Importance for Regressors"},{"location":"user_guide/evaluate/feature_importance_permutation/#example-3-feature-importance-with-one-hot-encoded-features","text":"Upon one-hot encoding, a feature variable with 10 distinct categories will be split into 10 new feature columns (or 9, if you drop a redundant column). If we want to treat each new feature column as an individual feature variable, we can use the feature_importance_permutation as usual. This is illustrated in the example below.","title":"Example 3 -- Feature Importance With One-Hot-Encoded Features"},{"location":"user_guide/evaluate/feature_importance_permutation/#preparing-the-dataset","text":"Here, we look at a dataset that consists of one categorical feature ( 'categorical' ) and 3 numerical features ( 'measurement1' , 'measurement2' , and 'measurement3' ). import pandas as pd df_data = pd.read_csv('https://gist.githubusercontent.com/rasbt/b99bf69079bc0d601eeae8a49248d358/raw/a114be9801647ec5460089f3a9576713dabf5f1f/onehot-numeric-mixed-data.csv') df_data.head() .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } categorical measurement1 measurement2 label measurement3 0 F 1.428571 2.721313 0 2.089 1 R 0.685939 0.982976 0 0.637 2 P 1.055817 0.624210 0 0.226 3 S 0.995956 0.321101 0 0.138 4 R 1.376773 1.578309 0 0.478 from sklearn.model_selection import train_test_split df_X = df_data[['measurement1', 'measurement2', 'measurement3', 'categorical']] df_y = df_data['label'] df_X_train, df_X_test, df_y_train, df_y_test = train_test_split( df_X, df_y, test_size=0.33, random_state=42, stratify=df_y) Here, we do the one-hot encoding on the categorical feature and merge it back with the numerical columns: from sklearn.preprocessing import OneHotEncoder import numpy as np ohe = OneHotEncoder(drop='first') ohe.fit(df_X_train[['categorical']]) df_X_train_ohe = df_X_train.drop(columns=['categorical']) df_X_test_ohe = df_X_test.drop(columns=['categorical']) ohe_train = np.asarray(ohe.transform(df_X_train[['categorical']]).todense()) ohe_test = np.asarray(ohe.transform(df_X_test[['categorical']]).todense()) X_train_ohe = np.hstack((df_X_train_ohe.values, ohe_train)) X_test_ohe = np.hstack((df_X_test_ohe.values, ohe_test)) # look at first 3 rows print(X_train_ohe[:3]) [[0.65747208 0.95105388 0.36 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 1. 0. 0. 0. 0. 0. ] [1.17503636 1.01094494 0.653 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 1. 0. 0. 0. 0. ] [1.25516647 0.67575824 0.176 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 1. 0. 0. 0. 0. 0. 0. ]]","title":"Preparing the Dataset"},{"location":"user_guide/evaluate/feature_importance_permutation/#fitting-a-baseline-model-for-analysis","text":"from sklearn.neural_network import MLPClassifier from sklearn.preprocessing import StandardScaler from sklearn.pipeline import make_pipeline from sklearn.model_selection import GridSearchCV pipe = make_pipeline(StandardScaler(), MLPClassifier(max_iter=10000, random_state=123)) params = { 'mlpclassifier__hidden_layer_sizes': [(30, 20, 10), (20, 10), (20,), (10,)], 'mlpclassifier__activation': ['tanh', 'relu'], 'mlpclassifier__solver': ['sgd'], 'mlpclassifier__alpha': [0.0001], 'mlpclassifier__learning_rate': ['adaptive'], } gs = GridSearchCV(estimator=pipe, param_grid=params, scoring='accuracy', refit=True, n_jobs=-1, cv=10) gs = gs.fit(X_train_ohe, df_y_train.values) model = gs.best_estimator_","title":"Fitting a Baseline Model for Analysis"},{"location":"user_guide/evaluate/feature_importance_permutation/#regular-permutation-importance","text":"Here, we compute the feature importance the regular way were each one-hot encoded feature is treated as an individiual variable. from mlxtend.evaluate import feature_importance_permutation imp_vals, imp_all = feature_importance_permutation( predict_method=model.predict, X=X_test_ohe, y=df_y_test.values, metric='accuracy', num_rounds=50, seed=1) feat_names_with_ohe = ['measurement1', 'measurement2', 'measurement3'] \\ + [f'categorical_ohe_{i}' for i in range(2, 20)] %matplotlib inline import matplotlib.pyplot as plt std = np.std(imp_all, axis=1) indices = np.argsort(imp_vals)[::-1] plt.figure() #plt.title(\"Feature importance via permutation importance w. std. dev.\") plt.bar(range(len(feat_names_with_ohe)), imp_vals[indices], yerr=std[indices]) plt.xticks(range(len(feat_names_with_ohe)), np.array(feat_names_with_ohe)[indices], rotation=90) plt.xlim([-1, len(feat_names_with_ohe)]) plt.show() However, notice that if we have a lot of categorical values, the feature importance of the individual binary features (after one hot encoding) are now hard to interpret. In certain cases, it would be desireable to treat the one-hot encoded binary features as a single variable in feature permutation importance evaluation. We can achieve this using feature groups.","title":"Regular Permutation Importance"},{"location":"user_guide/evaluate/feature_importance_permutation/#with-feature-groups","text":"In the example below, all the one-hot encoded variables are treated as a feature group. This means, they are are all shuffled and analyzed as a single feature inside the feature permutation importance analysis. feature_groups = [0, 1, 2, range(3, 21)] imp_vals, imp_all = feature_importance_permutation( predict_method=model.predict, X=X_test_ohe, y=df_y_test.values, metric='accuracy', num_rounds=50, feature_groups=feature_groups, seed=1) feature_names = ['measurement1', 'measurement2', 'measurement3', 'categorical'] std = np.std(imp_all, axis=1) indices = np.argsort(imp_vals)[::-1] plt.figure() plt.bar(range(len(feature_names)), imp_vals[indices], yerr=std[indices]) plt.xticks(range(len(feature_names)), np.array(feature_names)[indices], rotation=90) plt.xlim([-1, len(feature_names)]) plt.show()","title":"With Feature Groups"},{"location":"user_guide/evaluate/feature_importance_permutation/#api","text":"feature_importance_permutation(X, y, predict_method, metric, num_rounds=1, feature_groups=None, seed=None) Feature importance imputation via permutation importance Parameters X : NumPy array, shape = [n_samples, n_features] Dataset, where n_samples is the number of samples and n_features is the number of features. y : NumPy array, shape = [n_samples] Target values. predict_method : prediction function A callable function that predicts the target values from X. metric : str, callable The metric for evaluating the feature importance through permutation. By default, the strings 'accuracy' is recommended for classifiers and the string 'r2' is recommended for regressors. Optionally, a custom scoring function (e.g., metric=scoring_func ) that accepts two arguments, y_true and y_pred, which have similar shape to the y array. num_rounds : int (default=1) Number of rounds the feature columns are permuted to compute the permutation importance. feature_groups : list or None (default=None) Optional argument for treating certain features as a group. For example [1, 2, [3, 4, 5]] , which can be useful for interpretability, for example, if features 3, 4, 5 are one-hot encoded features. seed : int or None (default=None) Random seed for permuting the feature columns. Returns mean_importance_vals, all_importance_vals : NumPy arrays. The first array, mean_importance_vals has shape [n_features, ] and contains the importance values for all features. The shape of the second array is [n_features, num_rounds] and contains the feature importance for each repetition. If num_rounds=1, it contains the same values as the first array, mean_importance_vals. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/feature_importance_permutation/ ython ython","title":"API"},{"location":"user_guide/evaluate/ftest/","text":"ftest: F-test for classifier comparisons F-test for comparing the performance of multiple classifiers. from mlxtend.evaluate import ftest Overview In the context of evaluating machine learning models, the F-test by George W. Snedecor [1] can be regarded as analogous to Cochran's Q test that can be applied to evaluate multiple classifiers (i.e., whether their accuracies estimated on a test set differ) as described by Looney [2][3]. More formally, assume the task to test the null hypothesis that there is no difference between the classification accuracies [1]: p_i: H_0 = p_1 = p_2 = \\cdots = p_L. Let \\{C_1, \\dots , C_M\\} be a set of classifiers which have all been tested on the same dataset. If the M classifiers do not perform differently, then the F statistic is distributed according to an F distribution with (M-1) and (M-1)\\times n degrees of freedom, where n is the number of examples in the test set. The calculation of the F statistic consists of several components, which are listed below (adopted from [2]). We start by defining ACC_{avg} as the average of the accuracies of the different models ACC_{avg} = \\frac{1}{M}\\sum_{j=1}^M ACC_j. The sum of squares of the classifiers is then computed as SSA = n \\sum_{j=1}^{M} (G_j)^2 -n \\cdot M \\cdot ACC_{avg}, where G_j is the proportion of the n examples classified correctly by classifier j . The sum of squares for the objects is calculated as follows: SSB= \\frac{1}{M} \\sum_{j=1}^n (M_j)^2 - M\\cdot n \\cdot ACC_{avg}^2. Here, M_j is the number of classifiers out of M that correctly classified object \\mathbf{x}_j \\in \\mathbf{X}_{n} , where \\mathbf{X}_{n} = \\{\\mathbf{x}_1, ... \\mathbf{x}_{n}\\} is the test dataset on which the classifiers are tested on. Finally, we compute the total sum of squares, SST = M\\cdot n \\cdot ACC_{avg} (1 - ACC_{avg}), so that we then can compute the sum of squares for the classification--object interaction: SSAB = SST - SSA - SSB. To compute the F statistic, we next compute the mean SSA and mean SSAB values: MSA = \\frac{SSA}{M-1}, and MSAB = \\frac{SSAB}{(M-1) (n-1)}. From the MSA and MSAB, we can then calculate the F-value as F = \\frac{MSA}{MSAB}. After computing the F-value, we can then look up the p-value from a F-distribution table for the corresponding degrees of freedom or obtain it computationally from a cumulative F-distribution function. In practice, if we successfully rejected the null hypothesis at a previously chosen significance threshold, we could perform multiple post hoc pair-wise tests -- for example, McNemar tests with a Bonferroni correction -- to determine which pairs have different population proportions. References [1] Snedecor, George W. and Cochran, William G. (1989), Statistical Methods, Eighth Edition, Iowa State University Press. [2] Looney, Stephen W. \"A statistical technique for comparing the accuracies of several classifiers.\" Pattern Recognition Letters 8, no. 1 (1988): 5-9. [3] Kuncheva, Ludmila I. Combining pattern classifiers: methods and algorithms. John Wiley & Sons, 2004. Example 1 - F-test import numpy as np from mlxtend.evaluate import ftest ## Dataset: # ground truth labels of the test dataset: y_true = np.array([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]) # predictions by 3 classifiers (`y_model_1`, `y_model_2`, and `y_model_3`): y_model_1 = np.array([1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]) y_model_2 = np.array([1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]) y_model_3 = np.array([1, 1, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1]) Assuming a significance level \\alpha=0.05 , we can conduct Cochran's Q test as follows, to test the null hypothesis there is no difference between the classification accuracies, p_i: H_0 = p_1 = p_2 = \\cdots = p_L : f, p_value = ftest(y_true, y_model_1, y_model_2, y_model_3) print('F: %.3f' % f) print('p-value: %.3f' % p_value) F: 3.873 p-value: 0.022 Since the p-value is smaller than \\alpha , we can reject the null hypothesis and conclude that there is a difference between the classification accuracies. As mentioned in the introduction earlier, we could now perform multiple post hoc pair-wise tests -- for example, McNemar tests with a Bonferroni correction -- to determine which pairs have different population proportions. API ftest(y_target, y_model_predictions)* F-Test test to compare 2 or more models. Parameters y_target : array-like, shape=[n_samples] True class labels as 1D NumPy array. *y_model_predictions : array-likes, shape=[n_samples] Variable number of 2 or more arrays that contain the predicted class labels from models as 1D NumPy array. Returns f, p : float or None, float Returns the F-value and the p-value Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/ftest/","title":"ftest: F-test for classifier comparisons"},{"location":"user_guide/evaluate/ftest/#ftest-f-test-for-classifier-comparisons","text":"F-test for comparing the performance of multiple classifiers. from mlxtend.evaluate import ftest","title":"ftest: F-test for classifier comparisons"},{"location":"user_guide/evaluate/ftest/#overview","text":"In the context of evaluating machine learning models, the F-test by George W. Snedecor [1] can be regarded as analogous to Cochran's Q test that can be applied to evaluate multiple classifiers (i.e., whether their accuracies estimated on a test set differ) as described by Looney [2][3]. More formally, assume the task to test the null hypothesis that there is no difference between the classification accuracies [1]: p_i: H_0 = p_1 = p_2 = \\cdots = p_L. Let \\{C_1, \\dots , C_M\\} be a set of classifiers which have all been tested on the same dataset. If the M classifiers do not perform differently, then the F statistic is distributed according to an F distribution with (M-1) and (M-1)\\times n degrees of freedom, where n is the number of examples in the test set. The calculation of the F statistic consists of several components, which are listed below (adopted from [2]). We start by defining ACC_{avg} as the average of the accuracies of the different models ACC_{avg} = \\frac{1}{M}\\sum_{j=1}^M ACC_j. The sum of squares of the classifiers is then computed as SSA = n \\sum_{j=1}^{M} (G_j)^2 -n \\cdot M \\cdot ACC_{avg}, where G_j is the proportion of the n examples classified correctly by classifier j . The sum of squares for the objects is calculated as follows: SSB= \\frac{1}{M} \\sum_{j=1}^n (M_j)^2 - M\\cdot n \\cdot ACC_{avg}^2. Here, M_j is the number of classifiers out of M that correctly classified object \\mathbf{x}_j \\in \\mathbf{X}_{n} , where \\mathbf{X}_{n} = \\{\\mathbf{x}_1, ... \\mathbf{x}_{n}\\} is the test dataset on which the classifiers are tested on. Finally, we compute the total sum of squares, SST = M\\cdot n \\cdot ACC_{avg} (1 - ACC_{avg}), so that we then can compute the sum of squares for the classification--object interaction: SSAB = SST - SSA - SSB. To compute the F statistic, we next compute the mean SSA and mean SSAB values: MSA = \\frac{SSA}{M-1}, and MSAB = \\frac{SSAB}{(M-1) (n-1)}. From the MSA and MSAB, we can then calculate the F-value as F = \\frac{MSA}{MSAB}. After computing the F-value, we can then look up the p-value from a F-distribution table for the corresponding degrees of freedom or obtain it computationally from a cumulative F-distribution function. In practice, if we successfully rejected the null hypothesis at a previously chosen significance threshold, we could perform multiple post hoc pair-wise tests -- for example, McNemar tests with a Bonferroni correction -- to determine which pairs have different population proportions.","title":"Overview"},{"location":"user_guide/evaluate/ftest/#references","text":"[1] Snedecor, George W. and Cochran, William G. (1989), Statistical Methods, Eighth Edition, Iowa State University Press. [2] Looney, Stephen W. \"A statistical technique for comparing the accuracies of several classifiers.\" Pattern Recognition Letters 8, no. 1 (1988): 5-9. [3] Kuncheva, Ludmila I. Combining pattern classifiers: methods and algorithms. John Wiley & Sons, 2004.","title":"References"},{"location":"user_guide/evaluate/ftest/#example-1-f-test","text":"import numpy as np from mlxtend.evaluate import ftest ## Dataset: # ground truth labels of the test dataset: y_true = np.array([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]) # predictions by 3 classifiers (`y_model_1`, `y_model_2`, and `y_model_3`): y_model_1 = np.array([1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]) y_model_2 = np.array([1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]) y_model_3 = np.array([1, 1, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1]) Assuming a significance level \\alpha=0.05 , we can conduct Cochran's Q test as follows, to test the null hypothesis there is no difference between the classification accuracies, p_i: H_0 = p_1 = p_2 = \\cdots = p_L : f, p_value = ftest(y_true, y_model_1, y_model_2, y_model_3) print('F: %.3f' % f) print('p-value: %.3f' % p_value) F: 3.873 p-value: 0.022 Since the p-value is smaller than \\alpha , we can reject the null hypothesis and conclude that there is a difference between the classification accuracies. As mentioned in the introduction earlier, we could now perform multiple post hoc pair-wise tests -- for example, McNemar tests with a Bonferroni correction -- to determine which pairs have different population proportions.","title":"Example 1 - F-test"},{"location":"user_guide/evaluate/ftest/#api","text":"ftest(y_target, y_model_predictions)* F-Test test to compare 2 or more models. Parameters y_target : array-like, shape=[n_samples] True class labels as 1D NumPy array. *y_model_predictions : array-likes, shape=[n_samples] Variable number of 2 or more arrays that contain the predicted class labels from models as 1D NumPy array. Returns f, p : float or None, float Returns the F-value and the p-value Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/ftest/","title":"API"},{"location":"user_guide/evaluate/lift_score/","text":"lift_score: Lift score for classification and association rule mining Scoring function to compute the LIFT metric, the ratio of correctly predicted positive examples and the actual positive examples in the test dataset. from mlxtend.evaluate import lift_score Overview In the context of classification, lift [1] compares model predictions to randomly generated predictions. Lift is often used in marketing research combined with gain and lift charts as a visual aid [2]. For example, assuming a 10% customer response as a baseline, a lift value of 3 would correspond to a 30% customer response when using the predictive model. Note that lift has the range \\lbrack 0, \\infty \\rbrack . There are many strategies to compute lift , and below, we will illustrate the computation of the lift score using a classic confusion matrix. For instance, let's assume the following prediction and target labels, where \"1\" is the positive class: \\text{true labels}: [0, 0, 1, 0, 0, 1, 1, 1, 1, 1] \\text{prediction}: [1, 0, 1, 0, 0, 0, 0, 1, 0, 0] Then, our confusion matrix would look as follows: Based on the confusion matrix above, with \"1\" as positive label, we compute lift as follows: \\text{lift} = \\frac{(TP/(TP+FP)}{(TP+FN)/(TP+TN+FP+FN)} Plugging in the actual values from the example above, we arrive at the following lift value: \\frac{2/(2+1)}{(2+4)/(2+3+1+4)} = 1.1111111111111112 An alternative way to computing lift is by using the support metric [3]: \\text{lift} = \\frac{\\text{support}(\\text{true labels} \\cap \\text{prediction})}{\\text{support}(\\text{true labels}) \\times \\text{support}(\\text{prediction})}, Support is x / N , where x is the number of incidences of an observation and N is the total number of samples in the datset. \\text{true labels} \\cap \\text{prediction} are the true positives, true labels are true positives plus false negatives, and prediction are true positives plus false positives. Plugging the values from our example into the equation above, we arrive at: \\frac{2/10}{(6/10 \\times 3/10)} = 1.1111111111111112 References [1] S. Brin, R. Motwani, J. D. Ullman, and S. Tsur. Dynamic itemset counting and implication rules for market basket data . In Proc. of the ACM SIGMOD Int'l Conf. on Management of Data (ACM SIGMOD '97), pages 265-276, 1997. [2] https://www3.nd.edu/~busiforc/Lift_chart.html [3] https://en.wikipedia.org/wiki/Association_rule_learning#Support Example 1 - Computing Lift This examples demonstrates the basic use of the lift_score function using the example from the Overview section. import numpy as np from mlxtend.evaluate import lift_score y_target = np.array([0, 0, 1, 0, 0, 1, 1, 1, 1, 1]) y_predicted = np.array([1, 0, 1, 0, 0, 0, 0, 1, 0, 0]) lift_score(y_target, y_predicted) 1.1111111111111112 Example 2 - Using lift_score in GridSearch The lift_score function can also be used with scikit-learn objects, such as GridSearch : from sklearn.datasets import load_iris from sklearn.model_selection import train_test_split from sklearn.model_selection import GridSearchCV from sklearn.svm import SVC from sklearn.metrics import make_scorer # make custom scorer lift_scorer = make_scorer(lift_score) iris = load_iris() X, y = iris.data, iris.target X_train, X_test, y_train, y_test = train_test_split( X, y, test_size=0.2, stratify=y, random_state=123) hyperparameters = [{'kernel': ['rbf'], 'gamma': [1e-3, 1e-4], 'C': [1, 10, 100, 1000]}, {'kernel': ['linear'], 'C': [1, 10, 100, 1000]}] clf = GridSearchCV(SVC(), hyperparameters, cv=10, scoring=lift_scorer) clf.fit(X_train, y_train) print(clf.best_score_) print(clf.best_params_) 3.0 {'gamma': 0.001, 'kernel': 'rbf', 'C': 1000} API lift_score(y_target, y_predicted, binary=True, positive_label=1) Lift measures the degree to which the predictions of a classification model are better than randomly-generated predictions. The in terms of True Positives (TP), True Negatives (TN), False Positives (FP), and False Negatives (FN), the lift score is computed as: [ TP/(TP+FN) ] / [ (TP+FP) / (TP+TN+FP+FN) ] Parameters y_target : array-like, shape=[n_samples] True class labels. y_predicted : array-like, shape=[n_samples] Predicted class labels. binary : bool (default: True) Maps a multi-class problem onto a binary, where the positive class is 1 and all other classes are 0. positive_label : int (default: 0) Class label of the positive class. Returns score : float Lift score in the range [0, \\infty ] Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/lift_score/","title":"lift_score: Lift score for classification and association rule mining"},{"location":"user_guide/evaluate/lift_score/#lift_score-lift-score-for-classification-and-association-rule-mining","text":"Scoring function to compute the LIFT metric, the ratio of correctly predicted positive examples and the actual positive examples in the test dataset. from mlxtend.evaluate import lift_score","title":"lift_score: Lift score for classification and association rule mining"},{"location":"user_guide/evaluate/lift_score/#overview","text":"In the context of classification, lift [1] compares model predictions to randomly generated predictions. Lift is often used in marketing research combined with gain and lift charts as a visual aid [2]. For example, assuming a 10% customer response as a baseline, a lift value of 3 would correspond to a 30% customer response when using the predictive model. Note that lift has the range \\lbrack 0, \\infty \\rbrack . There are many strategies to compute lift , and below, we will illustrate the computation of the lift score using a classic confusion matrix. For instance, let's assume the following prediction and target labels, where \"1\" is the positive class: \\text{true labels}: [0, 0, 1, 0, 0, 1, 1, 1, 1, 1] \\text{prediction}: [1, 0, 1, 0, 0, 0, 0, 1, 0, 0] Then, our confusion matrix would look as follows: Based on the confusion matrix above, with \"1\" as positive label, we compute lift as follows: \\text{lift} = \\frac{(TP/(TP+FP)}{(TP+FN)/(TP+TN+FP+FN)} Plugging in the actual values from the example above, we arrive at the following lift value: \\frac{2/(2+1)}{(2+4)/(2+3+1+4)} = 1.1111111111111112 An alternative way to computing lift is by using the support metric [3]: \\text{lift} = \\frac{\\text{support}(\\text{true labels} \\cap \\text{prediction})}{\\text{support}(\\text{true labels}) \\times \\text{support}(\\text{prediction})}, Support is x / N , where x is the number of incidences of an observation and N is the total number of samples in the datset. \\text{true labels} \\cap \\text{prediction} are the true positives, true labels are true positives plus false negatives, and prediction are true positives plus false positives. Plugging the values from our example into the equation above, we arrive at: \\frac{2/10}{(6/10 \\times 3/10)} = 1.1111111111111112","title":"Overview"},{"location":"user_guide/evaluate/lift_score/#references","text":"[1] S. Brin, R. Motwani, J. D. Ullman, and S. Tsur. Dynamic itemset counting and implication rules for market basket data . In Proc. of the ACM SIGMOD Int'l Conf. on Management of Data (ACM SIGMOD '97), pages 265-276, 1997. [2] https://www3.nd.edu/~busiforc/Lift_chart.html [3] https://en.wikipedia.org/wiki/Association_rule_learning#Support","title":"References"},{"location":"user_guide/evaluate/lift_score/#example-1-computing-lift","text":"This examples demonstrates the basic use of the lift_score function using the example from the Overview section. import numpy as np from mlxtend.evaluate import lift_score y_target = np.array([0, 0, 1, 0, 0, 1, 1, 1, 1, 1]) y_predicted = np.array([1, 0, 1, 0, 0, 0, 0, 1, 0, 0]) lift_score(y_target, y_predicted) 1.1111111111111112","title":"Example 1 - Computing Lift"},{"location":"user_guide/evaluate/lift_score/#example-2-using-lift_score-in-gridsearch","text":"The lift_score function can also be used with scikit-learn objects, such as GridSearch : from sklearn.datasets import load_iris from sklearn.model_selection import train_test_split from sklearn.model_selection import GridSearchCV from sklearn.svm import SVC from sklearn.metrics import make_scorer # make custom scorer lift_scorer = make_scorer(lift_score) iris = load_iris() X, y = iris.data, iris.target X_train, X_test, y_train, y_test = train_test_split( X, y, test_size=0.2, stratify=y, random_state=123) hyperparameters = [{'kernel': ['rbf'], 'gamma': [1e-3, 1e-4], 'C': [1, 10, 100, 1000]}, {'kernel': ['linear'], 'C': [1, 10, 100, 1000]}] clf = GridSearchCV(SVC(), hyperparameters, cv=10, scoring=lift_scorer) clf.fit(X_train, y_train) print(clf.best_score_) print(clf.best_params_) 3.0 {'gamma': 0.001, 'kernel': 'rbf', 'C': 1000}","title":"Example 2 - Using lift_score in GridSearch"},{"location":"user_guide/evaluate/lift_score/#api","text":"lift_score(y_target, y_predicted, binary=True, positive_label=1) Lift measures the degree to which the predictions of a classification model are better than randomly-generated predictions. The in terms of True Positives (TP), True Negatives (TN), False Positives (FP), and False Negatives (FN), the lift score is computed as: [ TP/(TP+FN) ] / [ (TP+FP) / (TP+TN+FP+FN) ] Parameters y_target : array-like, shape=[n_samples] True class labels. y_predicted : array-like, shape=[n_samples] Predicted class labels. binary : bool (default: True) Maps a multi-class problem onto a binary, where the positive class is 1 and all other classes are 0. positive_label : int (default: 0) Class label of the positive class. Returns score : float Lift score in the range [0, \\infty ] Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/lift_score/","title":"API"},{"location":"user_guide/evaluate/mcnemar/","text":"mcnemar: McNemar's test for classifier comparisons McNemar's test for paired nominal data from mlxtend.evaluate import mcnemar Overview McNemar's Test [1] (sometimes also called \"within-subjects chi-squared test\") is a statistical test for paired nominal data. In context of machine learning (or statistical) models, we can use McNemar's Test to compare the predictive accuracy of two models. McNemar's test is based on a 2 times 2 contingency table of the two model's predictions. McNemar's Test Statistic In McNemar's Test, we formulate the null hypothesis that the probabilities p(b) and p(c) are the same, or in simplified terms: None of the two models performs better than the other. Thus, the alternative hypothesis is that the performances of the two models are not equal. The McNemar test statistic (\"chi-squared\") can be computed as follows: \\chi^2 = \\frac{(b - c)^2}{(b + c)}, If the sum of cell c and b is sufficiently large, the \\chi^2 value follows a chi-squared distribution with one degree of freedom. After setting a significance threshold, e.g,. \\alpha=0.05 we can compute the p-value -- assuming that the null hypothesis is true, the p-value is the probability of observing this empirical (or a larger) chi-squared value. If the p-value is lower than our chosen significance level, we can reject the null hypothesis that the two model's performances are equal. Continuity Correction Approximately 1 year after Quinn McNemar published the McNemar Test [1], Edwards [2] proposed a continuity corrected version, which is the more commonly used variant today: \\chi^2 = \\frac{( \\mid b - c \\mid - 1)^2}{(b + c)}. Exact p-values As mentioned earlier, an exact binomial test is recommended for small sample sizes ( b + c < 25 [3]), since the chi-squared value may not be well-approximated by the chi-squared distribution. The exact p-value can be computed as follows: p = 2 \\sum^{n}_{i=b} \\binom{n}{i} 0.5^i (1 - 0.5)^{n-i}, where n = b + c , and the factor 2 is used to compute the two-sided p-value. Example For instance, given that 2 models have a accuracy of with a 99.7% and 99.6% a 2x2 contingency table can provide further insights for model selection. In both subfigure A and B, the predictive accuracies of the two models are as follows: model 1 accuracy: 9,960 / 10,000 = 99.6% model 2 accuracy: 9,970 / 10,000 = 99.7% Now, in subfigure A, we can see that model 2 got 11 predictions right that model 1 got wrong. Vice versa, model 2 got 1 prediction right that model 2 got wrong. Thus, based on this 11:1 ratio, we may conclude that model 2 performs substantially better than model 1. However, in subfigure B, the ratio is 25:15, which is less conclusive about which model is the better one to choose. In the following coding examples, we will use these 2 scenarios A and B to illustrate McNemar's test. References [1] McNemar, Quinn, 1947. \" Note on the sampling error of the difference between correlated proportions or percentages \". Psychometrika. 12 (2): 153\u2013157. [2] Edwards AL: Note on the \u201ccorrection for continuity\u201d in testing the significance of the difference between correlated proportions. Psychometrika. 1948, 13 (3): 185-187. 10.1007/BF02289261. [3] https://en.wikipedia.org/wiki/McNemar%27s_test Example 1 - Creating 2x2 contingency tables The mcnemar funtion expects a 2x2 contingency table as a NumPy array that is formatted as follows: Such a contingency matrix can be created by using the mcnemar_table function from mlxtend.evaluate . For example: import numpy as np from mlxtend.evaluate import mcnemar_table # The correct target (class) labels y_target = np.array([0, 0, 0, 0, 0, 1, 1, 1, 1, 1]) # Class labels predicted by model 1 y_model1 = np.array([0, 1, 0, 0, 0, 1, 1, 0, 0, 0]) # Class labels predicted by model 2 y_model2 = np.array([0, 0, 1, 1, 0, 1, 1, 0, 0, 0]) tb = mcnemar_table(y_target=y_target, y_model1=y_model1, y_model2=y_model2) print(tb) [[4 1] [2 3]] Example 2 - McNemar's Test for Scenario B No, let us continue with the example mentioned in the overview section and assume that we already computed the 2x2 contingency table: import numpy as np tb_b = np.array([[9945, 25], [15, 15]]) To test the null hypothesis that the predictive performance of two models are equal (using a significance level of \\alpha=0.05 ), we can conduct a corrected McNemar Test for computing the chi-squared and p-value as follows: from mlxtend.evaluate import mcnemar chi2, p = mcnemar(ary=tb_b, corrected=True) print('chi-squared:', chi2) print('p-value:', p) chi-squared: 2.025 p-value: 0.154728923485 Since the p-value is larger than our assumed significance threshold ( \\alpha=0.05 ), we cannot reject our null hypothesis and assume that there is no significant difference between the two predictive models. Example 3 - McNemar's Test for Scenario A In contrast to scenario B (Example 2), the sample size in scenario A is relatively small (b + c = 11 + 1 = 12) and smaller than the recommended 25 [3] to approximate the computed chi-square value by the chi-square distribution well. In this case, we need to compute the exact p-value from the binomial distribution: from mlxtend.evaluate import mcnemar import numpy as np tb_a = np.array([[9959, 11], [1, 29]]) chi2, p = mcnemar(ary=tb_a, exact=True) print('chi-squared:', chi2) print('p-value:', p) chi-squared: None p-value: 0.005859375 Assuming that we conducted this test also with a significance level of \\alpha=0.05 , we can reject the null-hypothesis that both models perform equally well on this dataset, since the p-value ( p \\approx 0.006 ) is smaller than \\alpha . API mcnemar(ary, corrected=True, exact=False) McNemar test for paired nominal data Parameters ary : array-like, shape=[2, 2] 2 x 2 contingency table (as returned by evaluate.mcnemar_table), where a: ary[0, 0]: # of samples that both models predicted correctly b: ary[0, 1]: # of samples that model 1 got right and model 2 got wrong c: ary[1, 0]: # of samples that model 2 got right and model 1 got wrong d: aryCell [1, 1]: # of samples that both models predicted incorrectly corrected : array-like, shape=[n_samples] (default: True) Uses Edward's continuity correction for chi-squared if True exact : bool, (default: False) If True , uses an exact binomial test comparing b to a binomial distribution with n = b + c and p = 0.5. It is highly recommended to use exact=True for sample sizes < 25 since chi-squared is not well-approximated by the chi-squared distribution! Returns chi2, p : float or None, float Returns the chi-squared value and the p-value; if exact=True (default: False ), chi2 is None Examples For usage examples, please see [https://rasbt.github.io/mlxtend/user_guide/evaluate/mcnemar/](https://rasbt.github.io/mlxtend/user_guide/evaluate/mcnemar/) ython","title":"mcnemar: McNemar's test for classifier comparisons"},{"location":"user_guide/evaluate/mcnemar/#mcnemar-mcnemars-test-for-classifier-comparisons","text":"McNemar's test for paired nominal data from mlxtend.evaluate import mcnemar","title":"mcnemar: McNemar's test for classifier comparisons"},{"location":"user_guide/evaluate/mcnemar/#overview","text":"McNemar's Test [1] (sometimes also called \"within-subjects chi-squared test\") is a statistical test for paired nominal data. In context of machine learning (or statistical) models, we can use McNemar's Test to compare the predictive accuracy of two models. McNemar's test is based on a 2 times 2 contingency table of the two model's predictions.","title":"Overview"},{"location":"user_guide/evaluate/mcnemar/#mcnemars-test-statistic","text":"In McNemar's Test, we formulate the null hypothesis that the probabilities p(b) and p(c) are the same, or in simplified terms: None of the two models performs better than the other. Thus, the alternative hypothesis is that the performances of the two models are not equal. The McNemar test statistic (\"chi-squared\") can be computed as follows: \\chi^2 = \\frac{(b - c)^2}{(b + c)}, If the sum of cell c and b is sufficiently large, the \\chi^2 value follows a chi-squared distribution with one degree of freedom. After setting a significance threshold, e.g,. \\alpha=0.05 we can compute the p-value -- assuming that the null hypothesis is true, the p-value is the probability of observing this empirical (or a larger) chi-squared value. If the p-value is lower than our chosen significance level, we can reject the null hypothesis that the two model's performances are equal.","title":"McNemar's Test Statistic"},{"location":"user_guide/evaluate/mcnemar/#continuity-correction","text":"Approximately 1 year after Quinn McNemar published the McNemar Test [1], Edwards [2] proposed a continuity corrected version, which is the more commonly used variant today: \\chi^2 = \\frac{( \\mid b - c \\mid - 1)^2}{(b + c)}.","title":"Continuity Correction"},{"location":"user_guide/evaluate/mcnemar/#exact-p-values","text":"As mentioned earlier, an exact binomial test is recommended for small sample sizes ( b + c < 25 [3]), since the chi-squared value may not be well-approximated by the chi-squared distribution. The exact p-value can be computed as follows: p = 2 \\sum^{n}_{i=b} \\binom{n}{i} 0.5^i (1 - 0.5)^{n-i}, where n = b + c , and the factor 2 is used to compute the two-sided p-value.","title":"Exact p-values"},{"location":"user_guide/evaluate/mcnemar/#example","text":"For instance, given that 2 models have a accuracy of with a 99.7% and 99.6% a 2x2 contingency table can provide further insights for model selection. In both subfigure A and B, the predictive accuracies of the two models are as follows: model 1 accuracy: 9,960 / 10,000 = 99.6% model 2 accuracy: 9,970 / 10,000 = 99.7% Now, in subfigure A, we can see that model 2 got 11 predictions right that model 1 got wrong. Vice versa, model 2 got 1 prediction right that model 2 got wrong. Thus, based on this 11:1 ratio, we may conclude that model 2 performs substantially better than model 1. However, in subfigure B, the ratio is 25:15, which is less conclusive about which model is the better one to choose. In the following coding examples, we will use these 2 scenarios A and B to illustrate McNemar's test.","title":"Example"},{"location":"user_guide/evaluate/mcnemar/#references","text":"[1] McNemar, Quinn, 1947. \" Note on the sampling error of the difference between correlated proportions or percentages \". Psychometrika. 12 (2): 153\u2013157. [2] Edwards AL: Note on the \u201ccorrection for continuity\u201d in testing the significance of the difference between correlated proportions. Psychometrika. 1948, 13 (3): 185-187. 10.1007/BF02289261. [3] https://en.wikipedia.org/wiki/McNemar%27s_test","title":"References"},{"location":"user_guide/evaluate/mcnemar/#example-1-creating-2x2-contingency-tables","text":"The mcnemar funtion expects a 2x2 contingency table as a NumPy array that is formatted as follows: Such a contingency matrix can be created by using the mcnemar_table function from mlxtend.evaluate . For example: import numpy as np from mlxtend.evaluate import mcnemar_table # The correct target (class) labels y_target = np.array([0, 0, 0, 0, 0, 1, 1, 1, 1, 1]) # Class labels predicted by model 1 y_model1 = np.array([0, 1, 0, 0, 0, 1, 1, 0, 0, 0]) # Class labels predicted by model 2 y_model2 = np.array([0, 0, 1, 1, 0, 1, 1, 0, 0, 0]) tb = mcnemar_table(y_target=y_target, y_model1=y_model1, y_model2=y_model2) print(tb) [[4 1] [2 3]]","title":"Example 1 - Creating 2x2 contingency tables"},{"location":"user_guide/evaluate/mcnemar/#example-2-mcnemars-test-for-scenario-b","text":"No, let us continue with the example mentioned in the overview section and assume that we already computed the 2x2 contingency table: import numpy as np tb_b = np.array([[9945, 25], [15, 15]]) To test the null hypothesis that the predictive performance of two models are equal (using a significance level of \\alpha=0.05 ), we can conduct a corrected McNemar Test for computing the chi-squared and p-value as follows: from mlxtend.evaluate import mcnemar chi2, p = mcnemar(ary=tb_b, corrected=True) print('chi-squared:', chi2) print('p-value:', p) chi-squared: 2.025 p-value: 0.154728923485 Since the p-value is larger than our assumed significance threshold ( \\alpha=0.05 ), we cannot reject our null hypothesis and assume that there is no significant difference between the two predictive models.","title":"Example 2 - McNemar's Test for Scenario B"},{"location":"user_guide/evaluate/mcnemar/#example-3-mcnemars-test-for-scenario-a","text":"In contrast to scenario B (Example 2), the sample size in scenario A is relatively small (b + c = 11 + 1 = 12) and smaller than the recommended 25 [3] to approximate the computed chi-square value by the chi-square distribution well. In this case, we need to compute the exact p-value from the binomial distribution: from mlxtend.evaluate import mcnemar import numpy as np tb_a = np.array([[9959, 11], [1, 29]]) chi2, p = mcnemar(ary=tb_a, exact=True) print('chi-squared:', chi2) print('p-value:', p) chi-squared: None p-value: 0.005859375 Assuming that we conducted this test also with a significance level of \\alpha=0.05 , we can reject the null-hypothesis that both models perform equally well on this dataset, since the p-value ( p \\approx 0.006 ) is smaller than \\alpha .","title":"Example 3 - McNemar's Test for Scenario A"},{"location":"user_guide/evaluate/mcnemar/#api","text":"mcnemar(ary, corrected=True, exact=False) McNemar test for paired nominal data Parameters ary : array-like, shape=[2, 2] 2 x 2 contingency table (as returned by evaluate.mcnemar_table), where a: ary[0, 0]: # of samples that both models predicted correctly b: ary[0, 1]: # of samples that model 1 got right and model 2 got wrong c: ary[1, 0]: # of samples that model 2 got right and model 1 got wrong d: aryCell [1, 1]: # of samples that both models predicted incorrectly corrected : array-like, shape=[n_samples] (default: True) Uses Edward's continuity correction for chi-squared if True exact : bool, (default: False) If True , uses an exact binomial test comparing b to a binomial distribution with n = b + c and p = 0.5. It is highly recommended to use exact=True for sample sizes < 25 since chi-squared is not well-approximated by the chi-squared distribution! Returns chi2, p : float or None, float Returns the chi-squared value and the p-value; if exact=True (default: False ), chi2 is None Examples For usage examples, please see [https://rasbt.github.io/mlxtend/user_guide/evaluate/mcnemar/](https://rasbt.github.io/mlxtend/user_guide/evaluate/mcnemar/) ython","title":"API"},{"location":"user_guide/evaluate/mcnemar_table/","text":"mcnemar_table: Contingency table for McNemar's test Function to compute a 2x2 contingency table for McNemar's Test from mlxtend.evaluate import mcnemar_table Overview contingency Table for McNemar's Test A 2x2 contingency table as being used in a McNemar's Test ( mlxtend.evaluate.mcnemar ) is a useful aid for comparing two different models. In contrast to a typical confusion matrix, this table compares two models to each other rather than showing the false positives, true positives, false negatives, and true negatives of a single model's predictions: For instance, given that 2 models have a accuracy of with a 99.7% and 99.6% a 2x2 contingency table can provide further insights for model selection. In both subfigure A and B, the predictive accuracies of the two models are as follows: model 1 accuracy: 9,960 / 10,000 = 99.6% model 2 accuracy: 9,970 / 10,000 = 99.7% Now, in subfigure A, we can see that model 2 got 11 predictions right that model 1 got wrong. Vice versa, model 2 got 1 prediction right that model 2 got wrong. Thus, based on this 11:1 ratio, we may conclude that model 2 performs substantially better than model 1. However, in subfigure B, the ratio is 25:15, which is less conclusive about which model is the better one to choose. References McNemar, Quinn, 1947. \" Note on the sampling error of the difference between correlated proportions or percentages \". Psychometrika. 12 (2): 153\u2013157. Edwards AL: Note on the \u201ccorrection for continuity\u201d in testing the significance of the difference between correlated proportions. Psychometrika. 1948, 13 (3): 185-187. 10.1007/BF02289261. https://en.wikipedia.org/wiki/McNemar%27s_test Example 2 - 2x2 contingency Table import numpy as np from mlxtend.evaluate import mcnemar_table y_true = np.array([0, 0, 0, 0, 0, 1, 1, 1, 1, 1]) y_mod1 = np.array([0, 1, 0, 0, 0, 1, 1, 0, 0, 0]) y_mod2 = np.array([0, 0, 1, 1, 0, 1, 1, 0, 0, 0]) tb = mcnemar_table(y_target=y_true, y_model1=y_mod1, y_model2=y_mod2) tb array([[4, 1], [2, 3]]) To visualize (and better interpret) the contingency table via matplotlib, we can use the checkerboard_plot function: from mlxtend.plotting import checkerboard_plot import matplotlib.pyplot as plt brd = checkerboard_plot(tb, figsize=(3, 3), fmt='%d', col_labels=['model 2 wrong', 'model 2 right'], row_labels=['model 1 wrong', 'model 1 right']) plt.show() API mcnemar_table(y_target, y_model1, y_model2) Compute a 2x2 contingency table for McNemar's test. Parameters y_target : array-like, shape=[n_samples] True class labels as 1D NumPy array. y_model1 : array-like, shape=[n_samples] Predicted class labels from model as 1D NumPy array. y_model2 : array-like, shape=[n_samples] Predicted class labels from model 2 as 1D NumPy array. Returns tb : array-like, shape=[2, 2] 2x2 contingency table with the following contents: a: tb[0, 0]: # of samples that both models predicted correctly b: tb[0, 1]: # of samples that model 1 got right and model 2 got wrong c: tb[1, 0]: # of samples that model 2 got right and model 1 got wrong d: tb[1, 1]: # of samples that both models predicted incorrectly Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/mcnemar_table/","title":"mcnemar_table: Contingency table for McNemar's test"},{"location":"user_guide/evaluate/mcnemar_table/#mcnemar_table-contingency-table-for-mcnemars-test","text":"Function to compute a 2x2 contingency table for McNemar's Test from mlxtend.evaluate import mcnemar_table","title":"mcnemar_table: Contingency table for McNemar's test"},{"location":"user_guide/evaluate/mcnemar_table/#overview","text":"","title":"Overview"},{"location":"user_guide/evaluate/mcnemar_table/#contingency-table-for-mcnemars-test","text":"A 2x2 contingency table as being used in a McNemar's Test ( mlxtend.evaluate.mcnemar ) is a useful aid for comparing two different models. In contrast to a typical confusion matrix, this table compares two models to each other rather than showing the false positives, true positives, false negatives, and true negatives of a single model's predictions: For instance, given that 2 models have a accuracy of with a 99.7% and 99.6% a 2x2 contingency table can provide further insights for model selection. In both subfigure A and B, the predictive accuracies of the two models are as follows: model 1 accuracy: 9,960 / 10,000 = 99.6% model 2 accuracy: 9,970 / 10,000 = 99.7% Now, in subfigure A, we can see that model 2 got 11 predictions right that model 1 got wrong. Vice versa, model 2 got 1 prediction right that model 2 got wrong. Thus, based on this 11:1 ratio, we may conclude that model 2 performs substantially better than model 1. However, in subfigure B, the ratio is 25:15, which is less conclusive about which model is the better one to choose.","title":"contingency Table for McNemar's Test"},{"location":"user_guide/evaluate/mcnemar_table/#references","text":"McNemar, Quinn, 1947. \" Note on the sampling error of the difference between correlated proportions or percentages \". Psychometrika. 12 (2): 153\u2013157. Edwards AL: Note on the \u201ccorrection for continuity\u201d in testing the significance of the difference between correlated proportions. Psychometrika. 1948, 13 (3): 185-187. 10.1007/BF02289261. https://en.wikipedia.org/wiki/McNemar%27s_test","title":"References"},{"location":"user_guide/evaluate/mcnemar_table/#example-2-2x2-contingency-table","text":"import numpy as np from mlxtend.evaluate import mcnemar_table y_true = np.array([0, 0, 0, 0, 0, 1, 1, 1, 1, 1]) y_mod1 = np.array([0, 1, 0, 0, 0, 1, 1, 0, 0, 0]) y_mod2 = np.array([0, 0, 1, 1, 0, 1, 1, 0, 0, 0]) tb = mcnemar_table(y_target=y_true, y_model1=y_mod1, y_model2=y_mod2) tb array([[4, 1], [2, 3]]) To visualize (and better interpret) the contingency table via matplotlib, we can use the checkerboard_plot function: from mlxtend.plotting import checkerboard_plot import matplotlib.pyplot as plt brd = checkerboard_plot(tb, figsize=(3, 3), fmt='%d', col_labels=['model 2 wrong', 'model 2 right'], row_labels=['model 1 wrong', 'model 1 right']) plt.show()","title":"Example 2 - 2x2 contingency Table"},{"location":"user_guide/evaluate/mcnemar_table/#api","text":"mcnemar_table(y_target, y_model1, y_model2) Compute a 2x2 contingency table for McNemar's test. Parameters y_target : array-like, shape=[n_samples] True class labels as 1D NumPy array. y_model1 : array-like, shape=[n_samples] Predicted class labels from model as 1D NumPy array. y_model2 : array-like, shape=[n_samples] Predicted class labels from model 2 as 1D NumPy array. Returns tb : array-like, shape=[2, 2] 2x2 contingency table with the following contents: a: tb[0, 0]: # of samples that both models predicted correctly b: tb[0, 1]: # of samples that model 1 got right and model 2 got wrong c: tb[1, 0]: # of samples that model 2 got right and model 1 got wrong d: tb[1, 1]: # of samples that both models predicted incorrectly Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/mcnemar_table/","title":"API"},{"location":"user_guide/evaluate/mcnemar_tables/","text":"mcnemar_tables: contingency tables for McNemar's test and Cochran's Q test Function to compute a 2x2 contingency tables for McNemar's Test and Cochran's Q Test from mlxtend.evaluate import mcnemar_tables Overview contingency Tables A 2x2 contingency table as being used in a McNemar's Test ( mlxtend.evaluate.mcnemar ) is a useful aid for comparing two different models. In contrast to a typical confusion matrix, this table compares two models to each other rather than showing the false positives, true positives, false negatives, and true negatives of a single model's predictions: For instance, given that 2 models have a accuracy of with a 99.7% and 99.6% a 2x2 contingency table can provide further insights for model selection. In both subfigure A and B, the predictive accuracies of the two models are as follows: model 1 accuracy: 9,960 / 10,000 = 99.6% model 2 accuracy: 9,970 / 10,000 = 99.7% Now, in subfigure A, we can see that model 2 got 11 predictions right that model 1 got wrong. Vice versa, model 2 got 1 prediction right that model 2 got wrong. Thus, based on this 11:1 ratio, we may conclude that model 2 performs substantially better than model 1. However, in subfigure B, the ratio is 25:15, which is less conclusive about which model is the better one to choose. References McNemar, Quinn, 1947. \" Note on the sampling error of the difference between correlated proportions or percentages \". Psychometrika. 12 (2): 153\u2013157. Edwards AL: Note on the \u201ccorrection for continuity\u201d in testing the significance of the difference between correlated proportions. Psychometrika. 1948, 13 (3): 185-187. 10.1007/BF02289261. https://en.wikipedia.org/wiki/McNemar%27s_test Example 1 - Single 2x2 contingency Table import numpy as np from mlxtend.evaluate import mcnemar_tables y_true = np.array([0, 0, 0, 0, 0, 1, 1, 1, 1, 1]) y_mod0 = np.array([0, 1, 0, 0, 0, 1, 1, 0, 0, 0]) y_mod1 = np.array([0, 0, 1, 1, 0, 1, 1, 0, 0, 0]) tb = mcnemar_tables(y_true, y_mod0, y_mod1) tb {'model_0 vs model_1': array([[ 4., 1.], [ 2., 3.]])} To visualize (and better interpret) the contingency table via matplotlib, we can use the checkerboard_plot function: from mlxtend.plotting import checkerboard_plot import matplotlib.pyplot as plt brd = checkerboard_plot(tb['model_0 vs model_1'], figsize=(3, 3), fmt='%d', col_labels=['model 2 wrong', 'model 2 right'], row_labels=['model 1 wrong', 'model 1 right']) plt.show() Example 2 - Multiple 2x2 contingency Tables If more than two models are provided as input to the mcnemar_tables function, a 2x2 contingency table will be created for each pair of models: import numpy as np from mlxtend.evaluate import mcnemar_tables y_true = np.array([0, 0, 0, 0, 0, 1, 1, 1, 1, 1]) y_mod0 = np.array([0, 1, 0, 0, 0, 1, 1, 0, 0, 0]) y_mod1 = np.array([0, 0, 1, 1, 0, 1, 1, 0, 0, 0]) y_mod2 = np.array([0, 0, 1, 1, 0, 1, 1, 0, 1, 0]) tb = mcnemar_tables(y_true, y_mod0, y_mod1, y_mod2) for key, value in tb.items(): print(key, '\\n', value, '\\n') model_0 vs model_1 [[ 4. 1.] [ 2. 3.]] model_0 vs model_2 [[ 4. 2.] [ 2. 2.]] model_1 vs model_2 [[ 5. 1.] [ 0. 4.]] API mcnemar_tables(y_target, y_model_predictions)* Compute multiple 2x2 contigency tables for McNemar's test or Cochran's Q test. Parameters y_target : array-like, shape=[n_samples] True class labels as 1D NumPy array. y_model_predictions : array-like, shape=[n_samples] Predicted class labels for a model. Returns tables : dict Dictionary of NumPy arrays with shape=[2, 2]. Each dictionary key names the two models to be compared based on the order the models were passed as *y_model_predictions . The number of dictionary entries is equal to the number of pairwise combinations between the m models, i.e., \"m choose 2.\" For example the following target array (containing the true labels) and 3 models y_true = np.array([0, 0, 0, 0, 0, 1, 1, 1, 1, 1]) y_mod0 = np.array([0, 1, 0, 0, 0, 1, 1, 0, 0, 0]) y_mod1 = np.array([0, 0, 1, 1, 0, 1, 1, 0, 0, 0]) y_mod2 = np.array([0, 1, 1, 1, 0, 1, 0, 0, 0, 0]) would result in the following dictionary: {'model_0 vs model_1': array([[ 4., 1.], [ 2., 3.]]), 'model_0 vs model_2': array([[ 3., 0.], [ 3., 4.]]), 'model_1 vs model_2': array([[ 3., 0.], [ 2., 5.]])} Each array is structured in the following way: tb[0, 0]: # of samples that both models predicted correctly tb[0, 1]: # of samples that model a got right and model b got wrong tb[1, 0]: # of samples that model b got right and model a got wrong tb[1, 1]: # of samples that both models predicted incorrectly Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/mcnemar_tables/ ython","title":"mcnemar_tables: contingency tables for McNemar's test and Cochran's Q test"},{"location":"user_guide/evaluate/mcnemar_tables/#mcnemar_tables-contingency-tables-for-mcnemars-test-and-cochrans-q-test","text":"Function to compute a 2x2 contingency tables for McNemar's Test and Cochran's Q Test from mlxtend.evaluate import mcnemar_tables","title":"mcnemar_tables: contingency tables for McNemar's test and Cochran's Q test"},{"location":"user_guide/evaluate/mcnemar_tables/#overview","text":"","title":"Overview"},{"location":"user_guide/evaluate/mcnemar_tables/#contingency-tables","text":"A 2x2 contingency table as being used in a McNemar's Test ( mlxtend.evaluate.mcnemar ) is a useful aid for comparing two different models. In contrast to a typical confusion matrix, this table compares two models to each other rather than showing the false positives, true positives, false negatives, and true negatives of a single model's predictions: For instance, given that 2 models have a accuracy of with a 99.7% and 99.6% a 2x2 contingency table can provide further insights for model selection. In both subfigure A and B, the predictive accuracies of the two models are as follows: model 1 accuracy: 9,960 / 10,000 = 99.6% model 2 accuracy: 9,970 / 10,000 = 99.7% Now, in subfigure A, we can see that model 2 got 11 predictions right that model 1 got wrong. Vice versa, model 2 got 1 prediction right that model 2 got wrong. Thus, based on this 11:1 ratio, we may conclude that model 2 performs substantially better than model 1. However, in subfigure B, the ratio is 25:15, which is less conclusive about which model is the better one to choose.","title":"contingency Tables"},{"location":"user_guide/evaluate/mcnemar_tables/#references","text":"McNemar, Quinn, 1947. \" Note on the sampling error of the difference between correlated proportions or percentages \". Psychometrika. 12 (2): 153\u2013157. Edwards AL: Note on the \u201ccorrection for continuity\u201d in testing the significance of the difference between correlated proportions. Psychometrika. 1948, 13 (3): 185-187. 10.1007/BF02289261. https://en.wikipedia.org/wiki/McNemar%27s_test","title":"References"},{"location":"user_guide/evaluate/mcnemar_tables/#example-1-single-2x2-contingency-table","text":"import numpy as np from mlxtend.evaluate import mcnemar_tables y_true = np.array([0, 0, 0, 0, 0, 1, 1, 1, 1, 1]) y_mod0 = np.array([0, 1, 0, 0, 0, 1, 1, 0, 0, 0]) y_mod1 = np.array([0, 0, 1, 1, 0, 1, 1, 0, 0, 0]) tb = mcnemar_tables(y_true, y_mod0, y_mod1) tb {'model_0 vs model_1': array([[ 4., 1.], [ 2., 3.]])} To visualize (and better interpret) the contingency table via matplotlib, we can use the checkerboard_plot function: from mlxtend.plotting import checkerboard_plot import matplotlib.pyplot as plt brd = checkerboard_plot(tb['model_0 vs model_1'], figsize=(3, 3), fmt='%d', col_labels=['model 2 wrong', 'model 2 right'], row_labels=['model 1 wrong', 'model 1 right']) plt.show()","title":"Example 1 - Single 2x2 contingency Table"},{"location":"user_guide/evaluate/mcnemar_tables/#example-2-multiple-2x2-contingency-tables","text":"If more than two models are provided as input to the mcnemar_tables function, a 2x2 contingency table will be created for each pair of models: import numpy as np from mlxtend.evaluate import mcnemar_tables y_true = np.array([0, 0, 0, 0, 0, 1, 1, 1, 1, 1]) y_mod0 = np.array([0, 1, 0, 0, 0, 1, 1, 0, 0, 0]) y_mod1 = np.array([0, 0, 1, 1, 0, 1, 1, 0, 0, 0]) y_mod2 = np.array([0, 0, 1, 1, 0, 1, 1, 0, 1, 0]) tb = mcnemar_tables(y_true, y_mod0, y_mod1, y_mod2) for key, value in tb.items(): print(key, '\\n', value, '\\n') model_0 vs model_1 [[ 4. 1.] [ 2. 3.]] model_0 vs model_2 [[ 4. 2.] [ 2. 2.]] model_1 vs model_2 [[ 5. 1.] [ 0. 4.]]","title":"Example 2 - Multiple 2x2 contingency Tables"},{"location":"user_guide/evaluate/mcnemar_tables/#api","text":"mcnemar_tables(y_target, y_model_predictions)* Compute multiple 2x2 contigency tables for McNemar's test or Cochran's Q test. Parameters y_target : array-like, shape=[n_samples] True class labels as 1D NumPy array. y_model_predictions : array-like, shape=[n_samples] Predicted class labels for a model. Returns tables : dict Dictionary of NumPy arrays with shape=[2, 2]. Each dictionary key names the two models to be compared based on the order the models were passed as *y_model_predictions . The number of dictionary entries is equal to the number of pairwise combinations between the m models, i.e., \"m choose 2.\" For example the following target array (containing the true labels) and 3 models y_true = np.array([0, 0, 0, 0, 0, 1, 1, 1, 1, 1]) y_mod0 = np.array([0, 1, 0, 0, 0, 1, 1, 0, 0, 0]) y_mod1 = np.array([0, 0, 1, 1, 0, 1, 1, 0, 0, 0]) y_mod2 = np.array([0, 1, 1, 1, 0, 1, 0, 0, 0, 0]) would result in the following dictionary: {'model_0 vs model_1': array([[ 4., 1.], [ 2., 3.]]), 'model_0 vs model_2': array([[ 3., 0.], [ 3., 4.]]), 'model_1 vs model_2': array([[ 3., 0.], [ 2., 5.]])} Each array is structured in the following way: tb[0, 0]: # of samples that both models predicted correctly tb[0, 1]: # of samples that model a got right and model b got wrong tb[1, 0]: # of samples that model b got right and model a got wrong tb[1, 1]: # of samples that both models predicted incorrectly Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/mcnemar_tables/ ython","title":"API"},{"location":"user_guide/evaluate/paired_ttest_5x2cv/","text":"paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons 5x2cv paired t test procedure to compare the performance of two models from mlxtend.evaluate import paired_ttest_5x2cv Overview The 5x2cv paired t test is a procedure for comparing the performance of two models (classifiers or regressors) that was proposed by Dietterich [1] to address shortcomings in other methods such as the resampled paired t test (see paired_ttest_resampled ) and the k-fold cross-validated paired t test (see paired_ttest_kfold_cv ). To explain how this method works, let's consider to estimator (e.g., classifiers) A and B. Further, we have a labeled dataset D . In the common hold-out method, we typically split the dataset into 2 parts: a training and a test set. In the 5x2cv paired t test, we repeat the splitting (50% training and 50% test data) 5 times. In each of the 5 iterations, we fit A and B to the training split and evaluate their performance ( p_A and p_B ) on the test split. Then, we rotate the training and test sets (the training set becomes the test set and vice versa) compute the performance again, which results in 2 performance difference measures: p^{(1)} = p^{(1)}_A - p^{(1)}_B and p^{(2)} = p^{(2)}_A - p^{(2)}_B. Then, we estimate the estimate mean and variance of the differences: \\overline{p} = \\frac{p^{(1)} + p^{(2)}}{2} and s^2 = (p^{(1)} - \\overline{p})^2 + (p^{(2)} - \\overline{p})^2. The variance of the difference is computed for the 5 iterations and then used to compute the t statistic as follows: t = \\frac{p_1^{(1)}}{\\sqrt{(1/5) \\sum_{i=1}^{5}s_i^2}}, where p_1^{(1)} is the p_1 from the very first iteration. The t statistic, assuming that it approximately follows as t distribution with 5 degrees of freedom, under the null hypothesis that the models A and B have equal performance. Using the t statistic, the p value can be computed and compared with a previously chosen significance level, e.g., \\alpha=0.05 . If the p value is smaller than \\alpha , we reject the null hypothesis and accept that there is a significant difference in the two models. References [1] Dietterich TG (1998) Approximate Statistical Tests for Comparing Supervised Classification Learning Algorithms. Neural Comput 10:1895\u20131923. Example 1 - 5x2cv paired t test Assume we want to compare two classification algorithms, logistic regression and a decision tree algorithm: from sklearn.linear_model import LogisticRegression from sklearn.tree import DecisionTreeClassifier from mlxtend.data import iris_data from sklearn.model_selection import train_test_split X, y = iris_data() clf1 = LogisticRegression(random_state=1) clf2 = DecisionTreeClassifier(random_state=1) X_train, X_test, y_train, y_test = \\ train_test_split(X, y, test_size=0.25, random_state=123) score1 = clf1.fit(X_train, y_train).score(X_test, y_test) score2 = clf2.fit(X_train, y_train).score(X_test, y_test) print('Logistic regression accuracy: %.2f%%' % (score1*100)) print('Decision tree accuracy: %.2f%%' % (score2*100)) Logistic regression accuracy: 97.37% Decision tree accuracy: 94.74% Note that these accuracy values are not used in the paired t test procedure as new test/train splits are generated during the resampling procedure, the values above are just serving the purpose of intuition. Now, let's assume a significance threshold of \\alpha=0.05 for rejecting the null hypothesis that both algorithms perform equally well on the dataset and conduct the 5x2cv t test: from mlxtend.evaluate import paired_ttest_5x2cv t, p = paired_ttest_5x2cv(estimator1=clf1, estimator2=clf2, X=X, y=y, random_seed=1) print('t statistic: %.3f' % t) print('p value: %.3f' % p) t statistic: -1.539 p value: 0.184 Since p > \\alpha , we cannot reject the null hypothesis and may conclude that the performance of the two algorithms is not significantly different. While it is generally not recommended to apply statistical tests multiple times without correction for multiple hypothesis testing, let us take a look at an example where the decision tree algorithm is limited to producing a very simple decision boundary that would result in a relatively bad performance: clf2 = DecisionTreeClassifier(random_state=1, max_depth=1) score2 = clf2.fit(X_train, y_train).score(X_test, y_test) print('Decision tree accuracy: %.2f%%' % (score2*100)) t, p = paired_ttest_5x2cv(estimator1=clf1, estimator2=clf2, X=X, y=y, random_seed=1) print('t statistic: %.3f' % t) print('p value: %.3f' % p) Decision tree accuracy: 63.16% t statistic: 5.386 p value: 0.003 Assuming that we conducted this test also with a significance level of \\alpha=0.05 , we can reject the null-hypothesis that both models perform equally well on this dataset, since the p-value ( p < 0.001 ) is smaller than \\alpha . API paired_ttest_5x2cv(estimator1, estimator2, X, y, scoring=None, random_seed=None) Implements the 5x2cv paired t test proposed by Dieterrich (1998) to compare the performance of two models. Parameters estimator1 : scikit-learn classifier or regressor estimator2 : scikit-learn classifier or regressor X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values. scoring : str, callable, or None (default: None) If None (default), uses 'accuracy' for sklearn classifiers and 'r2' for sklearn regressors. If str, uses a sklearn scoring metric string identifier, for example {accuracy, f1, precision, recall, roc_auc} for classifiers, {'mean_absolute_error', 'mean_squared_error'/'neg_mean_squared_error', 'median_absolute_error', 'r2'} for regressors. If a callable object or function is provided, it has to be conform with sklearn's signature scorer(estimator, X, y) ; see https://scikit-learn.org/stable/modules/generated/sklearn.metrics.make_scorer.html for more information. random_seed : int or None (default: None) Random seed for creating the test/train splits. Returns t : float The t-statistic pvalue : float Two-tailed p-value. If the chosen significance level is larger than the p-value, we reject the null hypothesis and accept that there are significant differences in the two compared models. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/paired_ttest_5x2cv/","title":"paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons"},{"location":"user_guide/evaluate/paired_ttest_5x2cv/#paired_ttest_5x2cv-5x2cv-paired-t-test-for-classifier-comparisons","text":"5x2cv paired t test procedure to compare the performance of two models from mlxtend.evaluate import paired_ttest_5x2cv","title":"paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons"},{"location":"user_guide/evaluate/paired_ttest_5x2cv/#overview","text":"The 5x2cv paired t test is a procedure for comparing the performance of two models (classifiers or regressors) that was proposed by Dietterich [1] to address shortcomings in other methods such as the resampled paired t test (see paired_ttest_resampled ) and the k-fold cross-validated paired t test (see paired_ttest_kfold_cv ). To explain how this method works, let's consider to estimator (e.g., classifiers) A and B. Further, we have a labeled dataset D . In the common hold-out method, we typically split the dataset into 2 parts: a training and a test set. In the 5x2cv paired t test, we repeat the splitting (50% training and 50% test data) 5 times. In each of the 5 iterations, we fit A and B to the training split and evaluate their performance ( p_A and p_B ) on the test split. Then, we rotate the training and test sets (the training set becomes the test set and vice versa) compute the performance again, which results in 2 performance difference measures: p^{(1)} = p^{(1)}_A - p^{(1)}_B and p^{(2)} = p^{(2)}_A - p^{(2)}_B. Then, we estimate the estimate mean and variance of the differences: \\overline{p} = \\frac{p^{(1)} + p^{(2)}}{2} and s^2 = (p^{(1)} - \\overline{p})^2 + (p^{(2)} - \\overline{p})^2. The variance of the difference is computed for the 5 iterations and then used to compute the t statistic as follows: t = \\frac{p_1^{(1)}}{\\sqrt{(1/5) \\sum_{i=1}^{5}s_i^2}}, where p_1^{(1)} is the p_1 from the very first iteration. The t statistic, assuming that it approximately follows as t distribution with 5 degrees of freedom, under the null hypothesis that the models A and B have equal performance. Using the t statistic, the p value can be computed and compared with a previously chosen significance level, e.g., \\alpha=0.05 . If the p value is smaller than \\alpha , we reject the null hypothesis and accept that there is a significant difference in the two models.","title":"Overview"},{"location":"user_guide/evaluate/paired_ttest_5x2cv/#references","text":"[1] Dietterich TG (1998) Approximate Statistical Tests for Comparing Supervised Classification Learning Algorithms. Neural Comput 10:1895\u20131923.","title":"References"},{"location":"user_guide/evaluate/paired_ttest_5x2cv/#example-1-5x2cv-paired-t-test","text":"Assume we want to compare two classification algorithms, logistic regression and a decision tree algorithm: from sklearn.linear_model import LogisticRegression from sklearn.tree import DecisionTreeClassifier from mlxtend.data import iris_data from sklearn.model_selection import train_test_split X, y = iris_data() clf1 = LogisticRegression(random_state=1) clf2 = DecisionTreeClassifier(random_state=1) X_train, X_test, y_train, y_test = \\ train_test_split(X, y, test_size=0.25, random_state=123) score1 = clf1.fit(X_train, y_train).score(X_test, y_test) score2 = clf2.fit(X_train, y_train).score(X_test, y_test) print('Logistic regression accuracy: %.2f%%' % (score1*100)) print('Decision tree accuracy: %.2f%%' % (score2*100)) Logistic regression accuracy: 97.37% Decision tree accuracy: 94.74% Note that these accuracy values are not used in the paired t test procedure as new test/train splits are generated during the resampling procedure, the values above are just serving the purpose of intuition. Now, let's assume a significance threshold of \\alpha=0.05 for rejecting the null hypothesis that both algorithms perform equally well on the dataset and conduct the 5x2cv t test: from mlxtend.evaluate import paired_ttest_5x2cv t, p = paired_ttest_5x2cv(estimator1=clf1, estimator2=clf2, X=X, y=y, random_seed=1) print('t statistic: %.3f' % t) print('p value: %.3f' % p) t statistic: -1.539 p value: 0.184 Since p > \\alpha , we cannot reject the null hypothesis and may conclude that the performance of the two algorithms is not significantly different. While it is generally not recommended to apply statistical tests multiple times without correction for multiple hypothesis testing, let us take a look at an example where the decision tree algorithm is limited to producing a very simple decision boundary that would result in a relatively bad performance: clf2 = DecisionTreeClassifier(random_state=1, max_depth=1) score2 = clf2.fit(X_train, y_train).score(X_test, y_test) print('Decision tree accuracy: %.2f%%' % (score2*100)) t, p = paired_ttest_5x2cv(estimator1=clf1, estimator2=clf2, X=X, y=y, random_seed=1) print('t statistic: %.3f' % t) print('p value: %.3f' % p) Decision tree accuracy: 63.16% t statistic: 5.386 p value: 0.003 Assuming that we conducted this test also with a significance level of \\alpha=0.05 , we can reject the null-hypothesis that both models perform equally well on this dataset, since the p-value ( p < 0.001 ) is smaller than \\alpha .","title":"Example 1 - 5x2cv paired t test"},{"location":"user_guide/evaluate/paired_ttest_5x2cv/#api","text":"paired_ttest_5x2cv(estimator1, estimator2, X, y, scoring=None, random_seed=None) Implements the 5x2cv paired t test proposed by Dieterrich (1998) to compare the performance of two models. Parameters estimator1 : scikit-learn classifier or regressor estimator2 : scikit-learn classifier or regressor X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values. scoring : str, callable, or None (default: None) If None (default), uses 'accuracy' for sklearn classifiers and 'r2' for sklearn regressors. If str, uses a sklearn scoring metric string identifier, for example {accuracy, f1, precision, recall, roc_auc} for classifiers, {'mean_absolute_error', 'mean_squared_error'/'neg_mean_squared_error', 'median_absolute_error', 'r2'} for regressors. If a callable object or function is provided, it has to be conform with sklearn's signature scorer(estimator, X, y) ; see https://scikit-learn.org/stable/modules/generated/sklearn.metrics.make_scorer.html for more information. random_seed : int or None (default: None) Random seed for creating the test/train splits. Returns t : float The t-statistic pvalue : float Two-tailed p-value. If the chosen significance level is larger than the p-value, we reject the null hypothesis and accept that there are significant differences in the two compared models. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/paired_ttest_5x2cv/","title":"API"},{"location":"user_guide/evaluate/paired_ttest_kfold_cv/","text":"paired_ttest_kfold_cv: K-fold cross-validated paired t test K-fold paired t test procedure to compare the performance of two models from mlxtend.evaluate import paired_ttest_kfold_cv Overview K-fold cross-validated paired t-test procedure is a common method for comparing the performance of two models (classifiers or regressors) and addresses some of the drawbacks of the resampled t-test procedure ; however, this method has still the problem that the training sets overlap and is not recommended to be used in practice [1], and techniques such as the paired_ttest_5x2cv should be used instead. To explain how this method works, let's consider to estimator (e.g., classifiers) A and B. Further, we have a labeled dataset D . In the common hold-out method, we typically split the dataset into 2 parts: a training and a test set. In the k-fold cross-validated paired t-test procedure, we split the test set into k parts of equal size, and each of these parts is then used for testing while the remaining k-1 parts (joined together) are used for training a classifier or regressor (i.e., the standard k-fold cross-validation procedure). In each k-fold cross-validation iteration, we then compute the difference in performance between A and B in each so that we obtain k difference measures. Now, by making the assumption that these k differences were independently drawn and follow an approximately normal distribution, we can compute the following t statistic with k-1 degrees of freedom according to Student's t test, under the null hypothesis that the models A and B have equal performance: t = \\frac{\\overline{p} \\sqrt{k}}{\\sqrt{\\sum_{i=1}^{k}(p^{(i) - \\overline{p}})^2 / (k-1)}}. Here, p^{(i)} computes the difference between the model performances in the i th iteration, p^{(i)} = p^{(i)}_A - p^{(i)}_B , and \\overline{p} represents the average difference between the classifier performances, \\overline{p} = \\frac{1}{k} \\sum^k_{i=1} p^{(i)} . Once we computed the t statistic we can compute the p value and compare it to our chosen significance level, e.g., \\alpha=0.05 . If the p value is smaller than \\alpha , we reject the null hypothesis and accept that there is a significant difference in the two models. The problem with this method, and the reason why it is not recommended to be used in practice, is that it violates an assumption of Student's t test [1]: the difference between the model performances ( p^{(i)} = p^{(i)}_A - p^{(i)}_B ) are not normal distributed because p^{(i)}_A and p^{(i)}_B are not independent the p^{(i)} 's themselves are not independent because training sets overlap References [1] Dietterich TG (1998) Approximate Statistical Tests for Comparing Supervised Classification Learning Algorithms. Neural Comput 10:1895\u20131923. Example 1 - K-fold cross-validated paired t test Assume we want to compare two classification algorithms, logistic regression and a decision tree algorithm: from sklearn.linear_model import LogisticRegression from sklearn.tree import DecisionTreeClassifier from mlxtend.data import iris_data from sklearn.model_selection import train_test_split X, y = iris_data() clf1 = LogisticRegression(random_state=1) clf2 = DecisionTreeClassifier(random_state=1) X_train, X_test, y_train, y_test = \\ train_test_split(X, y, test_size=0.25, random_state=123) score1 = clf1.fit(X_train, y_train).score(X_test, y_test) score2 = clf2.fit(X_train, y_train).score(X_test, y_test) print('Logistic regression accuracy: %.2f%%' % (score1*100)) print('Decision tree accuracy: %.2f%%' % (score2*100)) Logistic regression accuracy: 97.37% Decision tree accuracy: 94.74% Note that these accuracy values are not used in the paired t-test procedure as new test/train splits are generated during the resampling procedure, the values above are just serving the purpose of intuition. Now, let's assume a significance threshold of \\alpha=0.05 for rejecting the null hypothesis that both algorithms perform equally well on the dataset and conduct the k-fold cross-validated t-test: from mlxtend.evaluate import paired_ttest_kfold_cv t, p = paired_ttest_kfold_cv(estimator1=clf1, estimator2=clf2, X=X, y=y, random_seed=1) print('t statistic: %.3f' % t) print('p value: %.3f' % p) t statistic: -1.861 p value: 0.096 Since p > \\alpha , we cannot reject the null hypothesis and may conclude that the performance of the two algorithms is not significantly different. While it is generally not recommended to apply statistical tests multiple times without correction for multiple hypothesis testing, let us take a look at an example where the decision tree algorithm is limited to producing a very simple decision boundary that would result in a relatively bad performance: clf2 = DecisionTreeClassifier(random_state=1, max_depth=1) score2 = clf2.fit(X_train, y_train).score(X_test, y_test) print('Decision tree accuracy: %.2f%%' % (score2*100)) t, p = paired_ttest_kfold_cv(estimator1=clf1, estimator2=clf2, X=X, y=y, random_seed=1) print('t statistic: %.3f' % t) print('p value: %.3f' % p) Decision tree accuracy: 63.16% t statistic: 13.491 p value: 0.000 Assuming that we conducted this test also with a significance level of \\alpha=0.05 , we can reject the null-hypothesis that both models perform equally well on this dataset, since the p-value ( p < 0.001 ) is smaller than \\alpha . API paired_ttest_kfold_cv(estimator1, estimator2, X, y, cv=10, scoring=None, shuffle=False, random_seed=None) Implements the k-fold paired t test procedure to compare the performance of two models. Parameters estimator1 : scikit-learn classifier or regressor estimator2 : scikit-learn classifier or regressor X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values. cv : int (default: 10) Number of splits and iteration for the cross-validation procedure scoring : str, callable, or None (default: None) If None (default), uses 'accuracy' for sklearn classifiers and 'r2' for sklearn regressors. If str, uses a sklearn scoring metric string identifier, for example {accuracy, f1, precision, recall, roc_auc} for classifiers, {'mean_absolute_error', 'mean_squared_error'/'neg_mean_squared_error', 'median_absolute_error', 'r2'} for regressors. If a callable object or function is provided, it has to be conform with sklearn's signature scorer(estimator, X, y) ; see https://scikit-learn.org/stable/modules/generated/sklearn.metrics.make_scorer.html for more information. shuffle : bool (default: True) Whether to shuffle the dataset for generating the k-fold splits. random_seed : int or None (default: None) Random seed for shuffling the dataset for generating the k-fold splits. Ignored if shuffle=False. Returns t : float The t-statistic pvalue : float Two-tailed p-value. If the chosen significance level is larger than the p-value, we reject the null hypothesis and accept that there are significant differences in the two compared models. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/paired_ttest_kfold_cv/","title":"paired_ttest_kfold_cv: K-fold cross-validated paired t test"},{"location":"user_guide/evaluate/paired_ttest_kfold_cv/#paired_ttest_kfold_cv-k-fold-cross-validated-paired-t-test","text":"K-fold paired t test procedure to compare the performance of two models from mlxtend.evaluate import paired_ttest_kfold_cv","title":"paired_ttest_kfold_cv: K-fold cross-validated paired t test"},{"location":"user_guide/evaluate/paired_ttest_kfold_cv/#overview","text":"K-fold cross-validated paired t-test procedure is a common method for comparing the performance of two models (classifiers or regressors) and addresses some of the drawbacks of the resampled t-test procedure ; however, this method has still the problem that the training sets overlap and is not recommended to be used in practice [1], and techniques such as the paired_ttest_5x2cv should be used instead. To explain how this method works, let's consider to estimator (e.g., classifiers) A and B. Further, we have a labeled dataset D . In the common hold-out method, we typically split the dataset into 2 parts: a training and a test set. In the k-fold cross-validated paired t-test procedure, we split the test set into k parts of equal size, and each of these parts is then used for testing while the remaining k-1 parts (joined together) are used for training a classifier or regressor (i.e., the standard k-fold cross-validation procedure). In each k-fold cross-validation iteration, we then compute the difference in performance between A and B in each so that we obtain k difference measures. Now, by making the assumption that these k differences were independently drawn and follow an approximately normal distribution, we can compute the following t statistic with k-1 degrees of freedom according to Student's t test, under the null hypothesis that the models A and B have equal performance: t = \\frac{\\overline{p} \\sqrt{k}}{\\sqrt{\\sum_{i=1}^{k}(p^{(i) - \\overline{p}})^2 / (k-1)}}. Here, p^{(i)} computes the difference between the model performances in the i th iteration, p^{(i)} = p^{(i)}_A - p^{(i)}_B , and \\overline{p} represents the average difference between the classifier performances, \\overline{p} = \\frac{1}{k} \\sum^k_{i=1} p^{(i)} . Once we computed the t statistic we can compute the p value and compare it to our chosen significance level, e.g., \\alpha=0.05 . If the p value is smaller than \\alpha , we reject the null hypothesis and accept that there is a significant difference in the two models. The problem with this method, and the reason why it is not recommended to be used in practice, is that it violates an assumption of Student's t test [1]: the difference between the model performances ( p^{(i)} = p^{(i)}_A - p^{(i)}_B ) are not normal distributed because p^{(i)}_A and p^{(i)}_B are not independent the p^{(i)} 's themselves are not independent because training sets overlap","title":"Overview"},{"location":"user_guide/evaluate/paired_ttest_kfold_cv/#references","text":"[1] Dietterich TG (1998) Approximate Statistical Tests for Comparing Supervised Classification Learning Algorithms. Neural Comput 10:1895\u20131923.","title":"References"},{"location":"user_guide/evaluate/paired_ttest_kfold_cv/#example-1-k-fold-cross-validated-paired-t-test","text":"Assume we want to compare two classification algorithms, logistic regression and a decision tree algorithm: from sklearn.linear_model import LogisticRegression from sklearn.tree import DecisionTreeClassifier from mlxtend.data import iris_data from sklearn.model_selection import train_test_split X, y = iris_data() clf1 = LogisticRegression(random_state=1) clf2 = DecisionTreeClassifier(random_state=1) X_train, X_test, y_train, y_test = \\ train_test_split(X, y, test_size=0.25, random_state=123) score1 = clf1.fit(X_train, y_train).score(X_test, y_test) score2 = clf2.fit(X_train, y_train).score(X_test, y_test) print('Logistic regression accuracy: %.2f%%' % (score1*100)) print('Decision tree accuracy: %.2f%%' % (score2*100)) Logistic regression accuracy: 97.37% Decision tree accuracy: 94.74% Note that these accuracy values are not used in the paired t-test procedure as new test/train splits are generated during the resampling procedure, the values above are just serving the purpose of intuition. Now, let's assume a significance threshold of \\alpha=0.05 for rejecting the null hypothesis that both algorithms perform equally well on the dataset and conduct the k-fold cross-validated t-test: from mlxtend.evaluate import paired_ttest_kfold_cv t, p = paired_ttest_kfold_cv(estimator1=clf1, estimator2=clf2, X=X, y=y, random_seed=1) print('t statistic: %.3f' % t) print('p value: %.3f' % p) t statistic: -1.861 p value: 0.096 Since p > \\alpha , we cannot reject the null hypothesis and may conclude that the performance of the two algorithms is not significantly different. While it is generally not recommended to apply statistical tests multiple times without correction for multiple hypothesis testing, let us take a look at an example where the decision tree algorithm is limited to producing a very simple decision boundary that would result in a relatively bad performance: clf2 = DecisionTreeClassifier(random_state=1, max_depth=1) score2 = clf2.fit(X_train, y_train).score(X_test, y_test) print('Decision tree accuracy: %.2f%%' % (score2*100)) t, p = paired_ttest_kfold_cv(estimator1=clf1, estimator2=clf2, X=X, y=y, random_seed=1) print('t statistic: %.3f' % t) print('p value: %.3f' % p) Decision tree accuracy: 63.16% t statistic: 13.491 p value: 0.000 Assuming that we conducted this test also with a significance level of \\alpha=0.05 , we can reject the null-hypothesis that both models perform equally well on this dataset, since the p-value ( p < 0.001 ) is smaller than \\alpha .","title":"Example 1 - K-fold cross-validated paired t test"},{"location":"user_guide/evaluate/paired_ttest_kfold_cv/#api","text":"paired_ttest_kfold_cv(estimator1, estimator2, X, y, cv=10, scoring=None, shuffle=False, random_seed=None) Implements the k-fold paired t test procedure to compare the performance of two models. Parameters estimator1 : scikit-learn classifier or regressor estimator2 : scikit-learn classifier or regressor X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values. cv : int (default: 10) Number of splits and iteration for the cross-validation procedure scoring : str, callable, or None (default: None) If None (default), uses 'accuracy' for sklearn classifiers and 'r2' for sklearn regressors. If str, uses a sklearn scoring metric string identifier, for example {accuracy, f1, precision, recall, roc_auc} for classifiers, {'mean_absolute_error', 'mean_squared_error'/'neg_mean_squared_error', 'median_absolute_error', 'r2'} for regressors. If a callable object or function is provided, it has to be conform with sklearn's signature scorer(estimator, X, y) ; see https://scikit-learn.org/stable/modules/generated/sklearn.metrics.make_scorer.html for more information. shuffle : bool (default: True) Whether to shuffle the dataset for generating the k-fold splits. random_seed : int or None (default: None) Random seed for shuffling the dataset for generating the k-fold splits. Ignored if shuffle=False. Returns t : float The t-statistic pvalue : float Two-tailed p-value. If the chosen significance level is larger than the p-value, we reject the null hypothesis and accept that there are significant differences in the two compared models. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/paired_ttest_kfold_cv/","title":"API"},{"location":"user_guide/evaluate/paired_ttest_resampled/","text":"paired_ttest_resample: Resampled paired t test Resampled paired t test procedure to compare the performance of two models from mlxtend.evaluate import paired_ttest_resample Overview Resampled paired t test procedure (also called k-hold-out paired t test) is a popular method for comparing the performance of two models (classifiers or regressors); however, this method has many drawbacks and is not recommended to be used in practice [1], and techniques such as the paired_ttest_5x2cv should be used instead. To explain how this method works, let's consider to estimator (e.g., classifiers) A and B. Further, we have a labeled dataset D . In the common hold-out method, we typically split the dataset into 2 parts: a training and a test set. In the resampled paired t test procedure, we repeat this splitting procedure (with typically 2/3 training data and 1/3 test data) k times (usually 30). In each iteration, we train A and B on the training set and evaluate it on the test set. Then, we compute the difference in performance between A and B in each iteration so that we obtain k difference measures. Now, by making the assumption that these k differences were independently drawn and follow an approximately normal distribution, we can compute the following t statistic with k-1 degrees of freedom according to Student's t test, under the null hypothesis that the models A and B have equal performance: t = \\frac{\\overline{p} \\sqrt{k}}{\\sqrt{\\sum_{i=1}^{k}(p^{(i)} - \\overline{p})^2 / (k-1)}}. Here, p^{(i)} computes the difference between the model performances in the i th iteration, p^{(i)} = p^{(i)}_A - p^{(i)}_B , and \\overline{p} represents the average difference between the classifier performances, \\overline{p} = \\frac{1}{k} \\sum^k_{i=1} p^{(i)} . Once we computed the t statistic we can compute the p value and compare it to our chosen significance level, e.g., \\alpha=0.05 . If the p value is smaller than \\alpha , we reject the null hypothesis and accept that there is a significant difference in the two models. To summarize the procedure: i := 0 while i < k: split dataset into training and test subsets fit models A and B to the training set compute the performances of A and B on the test set record the performance difference between A and B i := i + 1 compute t-statistic compute p value from t-statistic with k-1 degrees of freedom compare p value to chosen significance threshold The problem with this method, and the reason why it is not recommended to be used in practice, is that it violates the assumptions of Student's t test [1]: the difference between the model performances ( p^{(i)} = p^{(i)}_A - p^{(i)}_B ) are not normal distributed because p^{(i)}_A and p^{(i)}_B are not independent the p^{(i)} 's themselves are not independent because of the overlapping test sets; also, test and training sets overlap as well References [1] Dietterich TG (1998) Approximate Statistical Tests for Comparing Supervised Classification Learning Algorithms. Neural Comput 10:1895\u20131923. Example 1 - Resampled paired t test Assume we want to compare two classification algorithms, logistic regression and a decision tree algorithm: from sklearn.linear_model import LogisticRegression from sklearn.tree import DecisionTreeClassifier from mlxtend.data import iris_data from sklearn.model_selection import train_test_split X, y = iris_data() clf1 = LogisticRegression(random_state=1) clf2 = DecisionTreeClassifier(random_state=1) X_train, X_test, y_train, y_test = \\ train_test_split(X, y, test_size=0.25, random_state=123) score1 = clf1.fit(X_train, y_train).score(X_test, y_test) score2 = clf2.fit(X_train, y_train).score(X_test, y_test) print('Logistic regression accuracy: %.2f%%' % (score1*100)) print('Decision tree accuracy: %.2f%%' % (score2*100)) Logistic regression accuracy: 97.37% Decision tree accuracy: 94.74% Note that these accuracy values are not used in the paired t test procedure as new test/train splits are generated during the resampling procedure, the values above are just serving the purpose of intuition. Now, let's assume a significance threshold of \\alpha=0.05 for rejecting the null hypothesis that both algorithms perform equally well on the dataset and conduct the paired sample t test: from mlxtend.evaluate import paired_ttest_resampled t, p = paired_ttest_resampled(estimator1=clf1, estimator2=clf2, X=X, y=y, random_seed=1) print('t statistic: %.3f' % t) print('p value: %.3f' % p) t statistic: -1.809 p value: 0.081 Since p > t , we cannot reject the null hypothesis and may conclude that the performance of the two algorithms is not significantly different. While it is generally not recommended to apply statistical tests multiple times without correction for multiple hypothesis testing, let us take a look at an example where the decision tree algorithm is limited to producing a very simple decision boundary that would result in a relatively bad performance: clf2 = DecisionTreeClassifier(random_state=1, max_depth=1) score2 = clf2.fit(X_train, y_train).score(X_test, y_test) print('Decision tree accuracy: %.2f%%' % (score2*100)) t, p = paired_ttest_resampled(estimator1=clf1, estimator2=clf2, X=X, y=y, random_seed=1) print('t statistic: %.3f' % t) print('p value: %.3f' % p) Decision tree accuracy: 63.16% t statistic: 39.214 p value: 0.000 Assuming that we conducted this test also with a significance level of \\alpha=0.05 , we can reject the null-hypothesis that both models perform equally well on this dataset, since the p-value ( p < 0.001 ) is smaller than \\alpha . API paired_ttest_resampled(estimator1, estimator2, X, y, num_rounds=30, test_size=0.3, scoring=None, random_seed=None) Implements the resampled paired t test procedure to compare the performance of two models (also called k-hold-out paired t test). Parameters estimator1 : scikit-learn classifier or regressor estimator2 : scikit-learn classifier or regressor X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values. num_rounds : int (default: 30) Number of resampling iterations (i.e., train/test splits) test_size : float or int (default: 0.3) If float, should be between 0.0 and 1.0 and represent the proportion of the dataset to use as a test set. If int, represents the absolute number of test exsamples. scoring : str, callable, or None (default: None) If None (default), uses 'accuracy' for sklearn classifiers and 'r2' for sklearn regressors. If str, uses a sklearn scoring metric string identifier, for example {accuracy, f1, precision, recall, roc_auc} for classifiers, {'mean_absolute_error', 'mean_squared_error'/'neg_mean_squared_error', 'median_absolute_error', 'r2'} for regressors. If a callable object or function is provided, it has to be conform with sklearn's signature scorer(estimator, X, y) ; see https://scikit-learn.org/stable/modules/generated/sklearn.metrics.make_scorer.html for more information. random_seed : int or None (default: None) Random seed for creating the test/train splits. Returns t : float The t-statistic pvalue : float Two-tailed p-value. If the chosen significance level is larger than the p-value, we reject the null hypothesis and accept that there are significant differences in the two compared models. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/paired_ttest_resampled/","title":"paired_ttest_resample: Resampled paired t test"},{"location":"user_guide/evaluate/paired_ttest_resampled/#paired_ttest_resample-resampled-paired-t-test","text":"Resampled paired t test procedure to compare the performance of two models from mlxtend.evaluate import paired_ttest_resample","title":"paired_ttest_resample: Resampled paired t test"},{"location":"user_guide/evaluate/paired_ttest_resampled/#overview","text":"Resampled paired t test procedure (also called k-hold-out paired t test) is a popular method for comparing the performance of two models (classifiers or regressors); however, this method has many drawbacks and is not recommended to be used in practice [1], and techniques such as the paired_ttest_5x2cv should be used instead. To explain how this method works, let's consider to estimator (e.g., classifiers) A and B. Further, we have a labeled dataset D . In the common hold-out method, we typically split the dataset into 2 parts: a training and a test set. In the resampled paired t test procedure, we repeat this splitting procedure (with typically 2/3 training data and 1/3 test data) k times (usually 30). In each iteration, we train A and B on the training set and evaluate it on the test set. Then, we compute the difference in performance between A and B in each iteration so that we obtain k difference measures. Now, by making the assumption that these k differences were independently drawn and follow an approximately normal distribution, we can compute the following t statistic with k-1 degrees of freedom according to Student's t test, under the null hypothesis that the models A and B have equal performance: t = \\frac{\\overline{p} \\sqrt{k}}{\\sqrt{\\sum_{i=1}^{k}(p^{(i)} - \\overline{p})^2 / (k-1)}}. Here, p^{(i)} computes the difference between the model performances in the i th iteration, p^{(i)} = p^{(i)}_A - p^{(i)}_B , and \\overline{p} represents the average difference between the classifier performances, \\overline{p} = \\frac{1}{k} \\sum^k_{i=1} p^{(i)} . Once we computed the t statistic we can compute the p value and compare it to our chosen significance level, e.g., \\alpha=0.05 . If the p value is smaller than \\alpha , we reject the null hypothesis and accept that there is a significant difference in the two models. To summarize the procedure: i := 0 while i < k: split dataset into training and test subsets fit models A and B to the training set compute the performances of A and B on the test set record the performance difference between A and B i := i + 1 compute t-statistic compute p value from t-statistic with k-1 degrees of freedom compare p value to chosen significance threshold The problem with this method, and the reason why it is not recommended to be used in practice, is that it violates the assumptions of Student's t test [1]: the difference between the model performances ( p^{(i)} = p^{(i)}_A - p^{(i)}_B ) are not normal distributed because p^{(i)}_A and p^{(i)}_B are not independent the p^{(i)} 's themselves are not independent because of the overlapping test sets; also, test and training sets overlap as well","title":"Overview"},{"location":"user_guide/evaluate/paired_ttest_resampled/#references","text":"[1] Dietterich TG (1998) Approximate Statistical Tests for Comparing Supervised Classification Learning Algorithms. Neural Comput 10:1895\u20131923.","title":"References"},{"location":"user_guide/evaluate/paired_ttest_resampled/#example-1-resampled-paired-t-test","text":"Assume we want to compare two classification algorithms, logistic regression and a decision tree algorithm: from sklearn.linear_model import LogisticRegression from sklearn.tree import DecisionTreeClassifier from mlxtend.data import iris_data from sklearn.model_selection import train_test_split X, y = iris_data() clf1 = LogisticRegression(random_state=1) clf2 = DecisionTreeClassifier(random_state=1) X_train, X_test, y_train, y_test = \\ train_test_split(X, y, test_size=0.25, random_state=123) score1 = clf1.fit(X_train, y_train).score(X_test, y_test) score2 = clf2.fit(X_train, y_train).score(X_test, y_test) print('Logistic regression accuracy: %.2f%%' % (score1*100)) print('Decision tree accuracy: %.2f%%' % (score2*100)) Logistic regression accuracy: 97.37% Decision tree accuracy: 94.74% Note that these accuracy values are not used in the paired t test procedure as new test/train splits are generated during the resampling procedure, the values above are just serving the purpose of intuition. Now, let's assume a significance threshold of \\alpha=0.05 for rejecting the null hypothesis that both algorithms perform equally well on the dataset and conduct the paired sample t test: from mlxtend.evaluate import paired_ttest_resampled t, p = paired_ttest_resampled(estimator1=clf1, estimator2=clf2, X=X, y=y, random_seed=1) print('t statistic: %.3f' % t) print('p value: %.3f' % p) t statistic: -1.809 p value: 0.081 Since p > t , we cannot reject the null hypothesis and may conclude that the performance of the two algorithms is not significantly different. While it is generally not recommended to apply statistical tests multiple times without correction for multiple hypothesis testing, let us take a look at an example where the decision tree algorithm is limited to producing a very simple decision boundary that would result in a relatively bad performance: clf2 = DecisionTreeClassifier(random_state=1, max_depth=1) score2 = clf2.fit(X_train, y_train).score(X_test, y_test) print('Decision tree accuracy: %.2f%%' % (score2*100)) t, p = paired_ttest_resampled(estimator1=clf1, estimator2=clf2, X=X, y=y, random_seed=1) print('t statistic: %.3f' % t) print('p value: %.3f' % p) Decision tree accuracy: 63.16% t statistic: 39.214 p value: 0.000 Assuming that we conducted this test also with a significance level of \\alpha=0.05 , we can reject the null-hypothesis that both models perform equally well on this dataset, since the p-value ( p < 0.001 ) is smaller than \\alpha .","title":"Example 1 - Resampled paired t test"},{"location":"user_guide/evaluate/paired_ttest_resampled/#api","text":"paired_ttest_resampled(estimator1, estimator2, X, y, num_rounds=30, test_size=0.3, scoring=None, random_seed=None) Implements the resampled paired t test procedure to compare the performance of two models (also called k-hold-out paired t test). Parameters estimator1 : scikit-learn classifier or regressor estimator2 : scikit-learn classifier or regressor X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values. num_rounds : int (default: 30) Number of resampling iterations (i.e., train/test splits) test_size : float or int (default: 0.3) If float, should be between 0.0 and 1.0 and represent the proportion of the dataset to use as a test set. If int, represents the absolute number of test exsamples. scoring : str, callable, or None (default: None) If None (default), uses 'accuracy' for sklearn classifiers and 'r2' for sklearn regressors. If str, uses a sklearn scoring metric string identifier, for example {accuracy, f1, precision, recall, roc_auc} for classifiers, {'mean_absolute_error', 'mean_squared_error'/'neg_mean_squared_error', 'median_absolute_error', 'r2'} for regressors. If a callable object or function is provided, it has to be conform with sklearn's signature scorer(estimator, X, y) ; see https://scikit-learn.org/stable/modules/generated/sklearn.metrics.make_scorer.html for more information. random_seed : int or None (default: None) Random seed for creating the test/train splits. Returns t : float The t-statistic pvalue : float Two-tailed p-value. If the chosen significance level is larger than the p-value, we reject the null hypothesis and accept that there are significant differences in the two compared models. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/paired_ttest_resampled/","title":"API"},{"location":"user_guide/evaluate/permutation_test/","text":"permutation_test: Permutation test for hypothesis testing An implementation of a permutation test for hypothesis testing -- testing the null hypothesis that two different groups come from the same distribution. from mlxtend.evaluate import permutation_test Overview Permutation tests (also called exact tests, randomization tests, or re-randomization tests) are nonparametric test procedures to test the null hypothesis that two different groups come from the same distribution. A permutation test can be used for significance or hypothesis testing (including A/B testing) without requiring to make any assumptions about the sampling distribution (e.g., it doesn't require the samples to be normal distributed). In this document, we will refer to the exact method as \"permutation test\" and the approximated method as \"randomization test.\" Permutation Test Mechanics Under the null hypothesis (treatment = control), any permutations are equally likely. (Note that there are (n+m)! permutations, where n is the number of records in the treatment sample, and m is the number of records in the control sample). For a two-sided test, we define the alternative hypothesis that the two samples are different (e.g., treatment != control). Compute the difference (here: mean) of sample x and sample y Combine all measurements into a single dataset Draw a permuted dataset from all possible permutations of the dataset in 2. Divide the permuted dataset into two datasets x' and y' of size n and m , respectively Compute the difference (here: mean) of sample x' and sample y' and record this difference Repeat steps 3-5 until all permutations are evaluated Return the p-value as the number of times the recorded differences were at least as extreme as the original difference from 1. and divide this number by the total number of permutations Here, the p-value is defined as the probability, given the null hypothesis (no difference between the samples) is true, that we obtain results that are at least as extreme as the results we observed (i.e., the sample difference from 1.). More formally, we can express the computation of the p-value as follows (adapted from [2]): p(t \\geq t_0) = \\frac{1}{(n+m)!} \\sum^{(n+m)!}_{j=1} I(t_j \\geq t_0), where t_0 is the observed value of the test statistic (1. in the list above), and t is the t-value, the statistic computed from the resamples (5.) t(x'_1, x'_2, ..., x'_n, y'_1, y'_2, ..., x'_m) = |\\bar{x'} - \\bar{y'}| , and I is the indicator function. Given a significance level that we specify prior to carrying out the permutation test (e.g., alpha=0.05), we fail to reject the null hypothesis if the p-value is greater than alpha. Note that if the number of permutation is large, sampling all permutation may not computationally be feasible. Thus, a common approximation is to perfom k rounds of permutations (where k is typically a value between 1000 and 2000). Paired Samples The permutation (/randomization) tests can also be performed for paired samples by setting paired=True . The paired tests are related to the regular permutation test procedure described above except that the permuted samples are created by randomly swapping the a treatment and a control data point within each pair. References [1] Efron, Bradley and Tibshirani, R. J., An introduction to the bootstrap, Chapman & Hall/CRC Monographs on Statistics & Applied Probability, 1994. [2] Unpingco, Jos\u00e9. Python for probability, statistics, and machine learning. Springer, 2016. [3] Pitman, E. J. G., Significance tests which may be applied to samples from any population, Royal Statistical Society Supplement, 1937, 4: 119-30 and 225-32 Example 1 -- Two-sided randomization test Perform a two-sided randomization test to test the null hypothesis that two groups, \"treatment\" and \"control\" come from the same distribution. We specify alpha=0.01 as our significance level. treatment = [ 28.44, 29.32, 31.22, 29.58, 30.34, 28.76, 29.21, 30.4 , 31.12, 31.78, 27.58, 31.57, 30.73, 30.43, 30.31, 30.32, 29.18, 29.52, 29.22, 30.56] control = [ 33.51, 30.63, 32.38, 32.52, 29.41, 30.93, 49.78, 28.96, 35.77, 31.42, 30.76, 30.6 , 23.64, 30.54, 47.78, 31.98, 34.52, 32.42, 31.32, 40.72] Since evaluating all possible permutations may take a while, we will use the approximation method (see the introduction for details) i.e., randomization test: from mlxtend.evaluate import permutation_test p_value = permutation_test(treatment, control, method='approximate', num_rounds=10000, seed=0) print(p_value) 0.0066993300669933005 Since p-value < alpha, we can reject the null hypothesis that the two samples come from the same distribution. Example 2 -- Permutation test for calculating the p-value for correlation analysis (Pearson's R) Note: this is a one-sided hypothesis testing as we conduct the permutation test as \"how many times obtain a correlation coefficient that is greater than the observed value?\" import numpy as np from mlxtend.evaluate import permutation_test x = np.array([1, 2, 3, 4, 5, 6]) y = np.array([2, 4, 1, 5, 6, 7]) print('Observed pearson R: %.2f' % np.corrcoef(x, y)[1][0]) p_value = permutation_test(x, y, method='exact', func=lambda x, y: np.corrcoef(x, y)[1][0], seed=0) print('P value: %.2f' % p_value) Observed pearson R: 0.81 P value: 0.10 Example 3 -- Paired two-sample randomization test Suppose we have a dataset consisting of the the depths (in meters) of seven lakes of Wisconsin: \\begin{array}{cccccccc} \\text { Year } & \\text{ Lake } 1 & 2 & 3 & 4 & 5 & 6 & 7 \\\\ \\hline 1980: & 3.67 & 1.72 & 3.46 & 2.60 & 2.03 & 2.10 & 3.01 \\\\ \\hline 1990: & 2.11 & 1.79 & 2.71 & 1.89 & 1.69 & 1.71 & 2.01 \\\\ \\hline \\end{array} We are interested in testing the null hypothesis that the lakes in 1980 and 1990 don't have a significantly different depth. For this paired two-sample test, we are conducting a randomization test for paired samples at a significance level of 0.05: from mlxtend.evaluate import permutation_test lakes_1980 = [3.67, 1.72, 3.46, 2.60, 2.03, 2.10, 3.01] lakes_1990 = [2.11, 1.79, 2.71, 1.89, 1.69, 1.71, 2.01] p_value = permutation_test( lakes_1980, lakes_1990, paired=True, method=\"approximate\", seed=0, num_rounds=100000 ) print('P value: %.3f' % p_value) P value: 0.031 Since the p value is smaller than the significance threshold of 0.05, we conclude that there is a significant difference between the lake depths in 1980 and 1990. API permutation_test(x, y, func='x_mean != y_mean', method='exact', num_rounds=1000, seed=None, paired=False) Nonparametric permutation test Parameters x : list or numpy array with shape (n_datapoints,) A list or 1D numpy array of the first sample (e.g., the treatment group). y : list or numpy array with shape (n_datapoints,) A list or 1D numpy array of the second sample (e.g., the control group). func : custom function or str (default: 'x_mean != y_mean') function to compute the statistic for the permutation test. - If 'x_mean != y_mean', uses func=lambda x, y: np.abs(np.mean(x) - np.mean(y))) for a two-sided test. - If 'x_mean > y_mean', uses func=lambda x, y: np.mean(x) - np.mean(y)) for a one-sided test. - If 'x_mean < y_mean', uses func=lambda x, y: np.mean(y) - np.mean(x)) for a one-sided test. method : 'approximate' or 'exact' (default: 'exact') If 'exact' (default), all possible permutations are considered. If 'approximate' the number of drawn samples is given by num_rounds . Note that 'exact' is typically not feasible unless the dataset size is relatively small. paired : bool If True, a paired test is performed by only exchanging each datapoint with its associate. num_rounds : int (default: 1000) The number of permutation samples if method='approximate' . seed : int or None (default: None) The random seed for generating permutation samples if method='approximate' . Returns p-value under the null hypothesis Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/permutation_test/","title":"permutation_test: Permutation test for hypothesis testing"},{"location":"user_guide/evaluate/permutation_test/#permutation_test-permutation-test-for-hypothesis-testing","text":"An implementation of a permutation test for hypothesis testing -- testing the null hypothesis that two different groups come from the same distribution. from mlxtend.evaluate import permutation_test","title":"permutation_test: Permutation test for hypothesis testing"},{"location":"user_guide/evaluate/permutation_test/#overview","text":"Permutation tests (also called exact tests, randomization tests, or re-randomization tests) are nonparametric test procedures to test the null hypothesis that two different groups come from the same distribution. A permutation test can be used for significance or hypothesis testing (including A/B testing) without requiring to make any assumptions about the sampling distribution (e.g., it doesn't require the samples to be normal distributed). In this document, we will refer to the exact method as \"permutation test\" and the approximated method as \"randomization test.\"","title":"Overview"},{"location":"user_guide/evaluate/permutation_test/#permutation-test-mechanics","text":"Under the null hypothesis (treatment = control), any permutations are equally likely. (Note that there are (n+m)! permutations, where n is the number of records in the treatment sample, and m is the number of records in the control sample). For a two-sided test, we define the alternative hypothesis that the two samples are different (e.g., treatment != control). Compute the difference (here: mean) of sample x and sample y Combine all measurements into a single dataset Draw a permuted dataset from all possible permutations of the dataset in 2. Divide the permuted dataset into two datasets x' and y' of size n and m , respectively Compute the difference (here: mean) of sample x' and sample y' and record this difference Repeat steps 3-5 until all permutations are evaluated Return the p-value as the number of times the recorded differences were at least as extreme as the original difference from 1. and divide this number by the total number of permutations Here, the p-value is defined as the probability, given the null hypothesis (no difference between the samples) is true, that we obtain results that are at least as extreme as the results we observed (i.e., the sample difference from 1.). More formally, we can express the computation of the p-value as follows (adapted from [2]): p(t \\geq t_0) = \\frac{1}{(n+m)!} \\sum^{(n+m)!}_{j=1} I(t_j \\geq t_0), where t_0 is the observed value of the test statistic (1. in the list above), and t is the t-value, the statistic computed from the resamples (5.) t(x'_1, x'_2, ..., x'_n, y'_1, y'_2, ..., x'_m) = |\\bar{x'} - \\bar{y'}| , and I is the indicator function. Given a significance level that we specify prior to carrying out the permutation test (e.g., alpha=0.05), we fail to reject the null hypothesis if the p-value is greater than alpha. Note that if the number of permutation is large, sampling all permutation may not computationally be feasible. Thus, a common approximation is to perfom k rounds of permutations (where k is typically a value between 1000 and 2000).","title":"Permutation Test Mechanics"},{"location":"user_guide/evaluate/permutation_test/#paired-samples","text":"The permutation (/randomization) tests can also be performed for paired samples by setting paired=True . The paired tests are related to the regular permutation test procedure described above except that the permuted samples are created by randomly swapping the a treatment and a control data point within each pair.","title":"Paired Samples"},{"location":"user_guide/evaluate/permutation_test/#references","text":"[1] Efron, Bradley and Tibshirani, R. J., An introduction to the bootstrap, Chapman & Hall/CRC Monographs on Statistics & Applied Probability, 1994. [2] Unpingco, Jos\u00e9. Python for probability, statistics, and machine learning. Springer, 2016. [3] Pitman, E. J. G., Significance tests which may be applied to samples from any population, Royal Statistical Society Supplement, 1937, 4: 119-30 and 225-32","title":"References"},{"location":"user_guide/evaluate/permutation_test/#example-1-two-sided-randomization-test","text":"Perform a two-sided randomization test to test the null hypothesis that two groups, \"treatment\" and \"control\" come from the same distribution. We specify alpha=0.01 as our significance level. treatment = [ 28.44, 29.32, 31.22, 29.58, 30.34, 28.76, 29.21, 30.4 , 31.12, 31.78, 27.58, 31.57, 30.73, 30.43, 30.31, 30.32, 29.18, 29.52, 29.22, 30.56] control = [ 33.51, 30.63, 32.38, 32.52, 29.41, 30.93, 49.78, 28.96, 35.77, 31.42, 30.76, 30.6 , 23.64, 30.54, 47.78, 31.98, 34.52, 32.42, 31.32, 40.72] Since evaluating all possible permutations may take a while, we will use the approximation method (see the introduction for details) i.e., randomization test: from mlxtend.evaluate import permutation_test p_value = permutation_test(treatment, control, method='approximate', num_rounds=10000, seed=0) print(p_value) 0.0066993300669933005 Since p-value < alpha, we can reject the null hypothesis that the two samples come from the same distribution.","title":"Example 1 -- Two-sided randomization test"},{"location":"user_guide/evaluate/permutation_test/#example-2-permutation-test-for-calculating-the-p-value-for-correlation-analysis-pearsons-r","text":"Note: this is a one-sided hypothesis testing as we conduct the permutation test as \"how many times obtain a correlation coefficient that is greater than the observed value?\" import numpy as np from mlxtend.evaluate import permutation_test x = np.array([1, 2, 3, 4, 5, 6]) y = np.array([2, 4, 1, 5, 6, 7]) print('Observed pearson R: %.2f' % np.corrcoef(x, y)[1][0]) p_value = permutation_test(x, y, method='exact', func=lambda x, y: np.corrcoef(x, y)[1][0], seed=0) print('P value: %.2f' % p_value) Observed pearson R: 0.81 P value: 0.10","title":"Example 2 -- Permutation test for calculating the p-value for correlation analysis (Pearson's R)"},{"location":"user_guide/evaluate/permutation_test/#example-3-paired-two-sample-randomization-test","text":"Suppose we have a dataset consisting of the the depths (in meters) of seven lakes of Wisconsin: \\begin{array}{cccccccc} \\text { Year } & \\text{ Lake } 1 & 2 & 3 & 4 & 5 & 6 & 7 \\\\ \\hline 1980: & 3.67 & 1.72 & 3.46 & 2.60 & 2.03 & 2.10 & 3.01 \\\\ \\hline 1990: & 2.11 & 1.79 & 2.71 & 1.89 & 1.69 & 1.71 & 2.01 \\\\ \\hline \\end{array} We are interested in testing the null hypothesis that the lakes in 1980 and 1990 don't have a significantly different depth. For this paired two-sample test, we are conducting a randomization test for paired samples at a significance level of 0.05: from mlxtend.evaluate import permutation_test lakes_1980 = [3.67, 1.72, 3.46, 2.60, 2.03, 2.10, 3.01] lakes_1990 = [2.11, 1.79, 2.71, 1.89, 1.69, 1.71, 2.01] p_value = permutation_test( lakes_1980, lakes_1990, paired=True, method=\"approximate\", seed=0, num_rounds=100000 ) print('P value: %.3f' % p_value) P value: 0.031 Since the p value is smaller than the significance threshold of 0.05, we conclude that there is a significant difference between the lake depths in 1980 and 1990.","title":"Example 3 -- Paired two-sample randomization test"},{"location":"user_guide/evaluate/permutation_test/#api","text":"permutation_test(x, y, func='x_mean != y_mean', method='exact', num_rounds=1000, seed=None, paired=False) Nonparametric permutation test Parameters x : list or numpy array with shape (n_datapoints,) A list or 1D numpy array of the first sample (e.g., the treatment group). y : list or numpy array with shape (n_datapoints,) A list or 1D numpy array of the second sample (e.g., the control group). func : custom function or str (default: 'x_mean != y_mean') function to compute the statistic for the permutation test. - If 'x_mean != y_mean', uses func=lambda x, y: np.abs(np.mean(x) - np.mean(y))) for a two-sided test. - If 'x_mean > y_mean', uses func=lambda x, y: np.mean(x) - np.mean(y)) for a one-sided test. - If 'x_mean < y_mean', uses func=lambda x, y: np.mean(y) - np.mean(x)) for a one-sided test. method : 'approximate' or 'exact' (default: 'exact') If 'exact' (default), all possible permutations are considered. If 'approximate' the number of drawn samples is given by num_rounds . Note that 'exact' is typically not feasible unless the dataset size is relatively small. paired : bool If True, a paired test is performed by only exchanging each datapoint with its associate. num_rounds : int (default: 1000) The number of permutation samples if method='approximate' . seed : int or None (default: None) The random seed for generating permutation samples if method='approximate' . Returns p-value under the null hypothesis Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/permutation_test/","title":"API"},{"location":"user_guide/evaluate/proportion_difference/","text":"proportion_difference: Difference of proportions test for classifiers. Test of the difference of proportions for classifier performance comparison. from mlxtend.evaluate import proportion_difference Overview There are several different statistical hypothesis testing frameworks that are being used in practice to compare the performance of classification models, including common methods such as difference of two proportions (here, the proportions are the estimated generalization accuracies from a test set), for which we can construct 95% confidence intervals based on the concept of the Normal Approximation to the Binomial that was covered in Part I. Performing a z-score test for two population proportions is inarguably the most straight-forward way to compare to models (but certainly not the best!): In a nutshell, if the 95% confidence intervals of the accuracies of two models do not overlap, we can reject the null hypothesis that the performance of both classifiers is equal at a confidence level of \\alpha=0.05 (or 5% probability). Violations of assumptions aside (for instance that the test set samples are not independent), as Thomas Dietterich noted based on empircal results in a simulated study [1], this test tends to have a high false positive rate (here: incorrectly detecting difference when there is none), which is among the reasons why it is not recommended in practice. Nonetheless, for the sake of completeness, and since it a commonly used method in practice, the general procedure is outlined below as follows (which also generally applies to the different hypothesis tests presented later): formulate the hypothesis to be tested (for instance, the null hypothesis stating that the proportions are the same; consequently, the alternative hypothesis that the proportions are different, if we use a two-tailed test); decide upon a significance threshold (for instance, if the probability of observing a difference more extreme than the one observed is more than 5%, then we plan to reject the null hypothesis); analyze the data, compute the test statistic (here: z-score), and compare its associated p-value (probability) to the previously determined significance threshold; based on the p-value and significance threshold, either accept or reject the null hypothesis at the given confidence level and interpret the results. The z-score is computed as the observed difference divided by the square root for their combined variances z = \\frac{ACC_1 - ACC_2}{\\sqrt{\\sigma_{1}^2 + \\sigma_{2}^2}}, where ACC_1 is the accuracy of one model and ACC_2 is the accuracy of a second model estimated from the test set. Recall that we computed the variance of the estimated of the estimated accuracy as \\sigma^2 = \\frac{ACC(1-ACC)}{n} in Part I and then computed the confidence interval (Normal Approximation Interval) as ACC \\pm z \\times \\sigma, where z=1.96 for a 95% confidence interval. Comparing the confidence intervals of two accuracy estimates and checking whether they overlap is then analogous to computing the z value for the difference in proportions and comparing the probability (p-value) to the chosen significance threshold. So, to compute the z-score directly for the difference of two proportions, ACC_1 and ACC_2 , we pool these proportions (assuming that ACC_1 and ACC_2 are the performances of two models estimated on two indendent test sets of size n_1 and n_2 , respectively), ACC_{1, 2} = \\frac{ACC_1 \\times n_1 + ACC_2 \\times n_2}{n_1 + n_2}, and compute the standard deviation as \\sigma_{1,2} = \\sqrt{\\frac{ACC_{1, 2} (1 - ACC_{1, 2})}{n_1 + n_2}}, such that we can compute the z-score, z = \\frac{ACC_1 - ACC_2}{\\sigma_{1,2}}. Since, due to using the same test set (and violating the independence assumption) we have n_1 = n_2 = n , so that we can simplify the z-score computation to z = \\frac{ACC_1 - ACC_2}{\\sqrt{2\\sigma^2}} = \\frac{ACC_1 - ACC_2}{\\sqrt{2\\cdot ACC_{1,2}(1-ACC_{1,2}))/n}}. where ACC_{1, 2} is simply (ACC_1 + ACC_2)/2 . In the second step, based on the computed z value (this assumes the the test errors are independent, which is usually violated in practice as we use the same test set) we can reject the null hypothesis that the a pair of models has equal performance (here, measured in \"classification aaccuracy\") at an \\alpha=0.05 level if z is greater than 1.96. Or if we want to put in the extra work, we can compute the area under the a standard normal cumulative distribution at the z-score threshold. If we find this p-value is smaller than a significance level we set prior to conducting the test, then we can reject the null hypothesis at that given significance level. The problem with this test though is that we use the same test set to compute the accuracy of the two classifiers; thus, it might be better to use a paired test such as a paired sample t-test, but a more robust alternative is the McNemar test. References [1] Dietterich, Thomas G. \"Approximate statistical tests for comparing supervised classification learning algorithms.\" Neural computation 10, no. 7 (1998): 1895-1923. Example 1 - Difference of Proportions As an example for applying this test, consider the following 2 model predictions: import numpy as np ## Dataset: # ground truth labels of the test dataset: y_true = np.array([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]) # predictions by 3 classifiers (`y_model_1`, `y_model_2`, and `y_model_3`): y_model_1 = np.array([1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]) y_model_2 = np.array([1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]) Assume, the test accuracies are as follows: acc_1 = np.sum(y_true == y_model_1) / y_true.shape[0] acc_2 = np.sum(y_true == y_model_2) / y_true.shape[0] print('Accuracy Model 1:', acc_1) print('Accuracy Model 2:', acc_2) Accuracy Model 1: 0.84 Accuracy Model 2: 0.92 Now, setting a significance threshold of \\alpha=0.05 and conducting the test from mlxtend.evaluate import proportion_difference z, p_value = proportion_difference(acc_1, acc_2, n_1=y_true.shape[0]) print('z: %.3f' % z) print('p-value: %.3f' % p_value) z: -1.754 p-value: 0.040 we find that there is a statistically significant difference between the model performances. It should be highlighted though that using this test, due to the typical independence violation of using the same test set as well as its high false positive rate, it is not recommended to use this test in practice. API proportion_difference(proportion_1, proportion_2, n_1, n_2=None) Computes the test statistic and p-value for a difference of proportions test. Parameters proportion_1 : float The first proportion proportion_2 : float The second proportion n_1 : int The sample size of the first test sample n_2 : int or None (default=None) The sample size of the second test sample. If None , n_1 = n_2 . Returns z, p : float or None, float Returns the z-score and the p-value Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/proportion_difference/","title":"proportion_difference: Difference of proportions test for classifiers."},{"location":"user_guide/evaluate/proportion_difference/#proportion_difference-difference-of-proportions-test-for-classifiers","text":"Test of the difference of proportions for classifier performance comparison. from mlxtend.evaluate import proportion_difference","title":"proportion_difference: Difference of proportions test for classifiers."},{"location":"user_guide/evaluate/proportion_difference/#overview","text":"There are several different statistical hypothesis testing frameworks that are being used in practice to compare the performance of classification models, including common methods such as difference of two proportions (here, the proportions are the estimated generalization accuracies from a test set), for which we can construct 95% confidence intervals based on the concept of the Normal Approximation to the Binomial that was covered in Part I. Performing a z-score test for two population proportions is inarguably the most straight-forward way to compare to models (but certainly not the best!): In a nutshell, if the 95% confidence intervals of the accuracies of two models do not overlap, we can reject the null hypothesis that the performance of both classifiers is equal at a confidence level of \\alpha=0.05 (or 5% probability). Violations of assumptions aside (for instance that the test set samples are not independent), as Thomas Dietterich noted based on empircal results in a simulated study [1], this test tends to have a high false positive rate (here: incorrectly detecting difference when there is none), which is among the reasons why it is not recommended in practice. Nonetheless, for the sake of completeness, and since it a commonly used method in practice, the general procedure is outlined below as follows (which also generally applies to the different hypothesis tests presented later): formulate the hypothesis to be tested (for instance, the null hypothesis stating that the proportions are the same; consequently, the alternative hypothesis that the proportions are different, if we use a two-tailed test); decide upon a significance threshold (for instance, if the probability of observing a difference more extreme than the one observed is more than 5%, then we plan to reject the null hypothesis); analyze the data, compute the test statistic (here: z-score), and compare its associated p-value (probability) to the previously determined significance threshold; based on the p-value and significance threshold, either accept or reject the null hypothesis at the given confidence level and interpret the results. The z-score is computed as the observed difference divided by the square root for their combined variances z = \\frac{ACC_1 - ACC_2}{\\sqrt{\\sigma_{1}^2 + \\sigma_{2}^2}}, where ACC_1 is the accuracy of one model and ACC_2 is the accuracy of a second model estimated from the test set. Recall that we computed the variance of the estimated of the estimated accuracy as \\sigma^2 = \\frac{ACC(1-ACC)}{n} in Part I and then computed the confidence interval (Normal Approximation Interval) as ACC \\pm z \\times \\sigma, where z=1.96 for a 95% confidence interval. Comparing the confidence intervals of two accuracy estimates and checking whether they overlap is then analogous to computing the z value for the difference in proportions and comparing the probability (p-value) to the chosen significance threshold. So, to compute the z-score directly for the difference of two proportions, ACC_1 and ACC_2 , we pool these proportions (assuming that ACC_1 and ACC_2 are the performances of two models estimated on two indendent test sets of size n_1 and n_2 , respectively), ACC_{1, 2} = \\frac{ACC_1 \\times n_1 + ACC_2 \\times n_2}{n_1 + n_2}, and compute the standard deviation as \\sigma_{1,2} = \\sqrt{\\frac{ACC_{1, 2} (1 - ACC_{1, 2})}{n_1 + n_2}}, such that we can compute the z-score, z = \\frac{ACC_1 - ACC_2}{\\sigma_{1,2}}. Since, due to using the same test set (and violating the independence assumption) we have n_1 = n_2 = n , so that we can simplify the z-score computation to z = \\frac{ACC_1 - ACC_2}{\\sqrt{2\\sigma^2}} = \\frac{ACC_1 - ACC_2}{\\sqrt{2\\cdot ACC_{1,2}(1-ACC_{1,2}))/n}}. where ACC_{1, 2} is simply (ACC_1 + ACC_2)/2 . In the second step, based on the computed z value (this assumes the the test errors are independent, which is usually violated in practice as we use the same test set) we can reject the null hypothesis that the a pair of models has equal performance (here, measured in \"classification aaccuracy\") at an \\alpha=0.05 level if z is greater than 1.96. Or if we want to put in the extra work, we can compute the area under the a standard normal cumulative distribution at the z-score threshold. If we find this p-value is smaller than a significance level we set prior to conducting the test, then we can reject the null hypothesis at that given significance level. The problem with this test though is that we use the same test set to compute the accuracy of the two classifiers; thus, it might be better to use a paired test such as a paired sample t-test, but a more robust alternative is the McNemar test.","title":"Overview"},{"location":"user_guide/evaluate/proportion_difference/#references","text":"[1] Dietterich, Thomas G. \"Approximate statistical tests for comparing supervised classification learning algorithms.\" Neural computation 10, no. 7 (1998): 1895-1923.","title":"References"},{"location":"user_guide/evaluate/proportion_difference/#example-1-difference-of-proportions","text":"As an example for applying this test, consider the following 2 model predictions: import numpy as np ## Dataset: # ground truth labels of the test dataset: y_true = np.array([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]) # predictions by 3 classifiers (`y_model_1`, `y_model_2`, and `y_model_3`): y_model_1 = np.array([1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]) y_model_2 = np.array([1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]) Assume, the test accuracies are as follows: acc_1 = np.sum(y_true == y_model_1) / y_true.shape[0] acc_2 = np.sum(y_true == y_model_2) / y_true.shape[0] print('Accuracy Model 1:', acc_1) print('Accuracy Model 2:', acc_2) Accuracy Model 1: 0.84 Accuracy Model 2: 0.92 Now, setting a significance threshold of \\alpha=0.05 and conducting the test from mlxtend.evaluate import proportion_difference z, p_value = proportion_difference(acc_1, acc_2, n_1=y_true.shape[0]) print('z: %.3f' % z) print('p-value: %.3f' % p_value) z: -1.754 p-value: 0.040 we find that there is a statistically significant difference between the model performances. It should be highlighted though that using this test, due to the typical independence violation of using the same test set as well as its high false positive rate, it is not recommended to use this test in practice.","title":"Example 1 - Difference of Proportions"},{"location":"user_guide/evaluate/proportion_difference/#api","text":"proportion_difference(proportion_1, proportion_2, n_1, n_2=None) Computes the test statistic and p-value for a difference of proportions test. Parameters proportion_1 : float The first proportion proportion_2 : float The second proportion n_1 : int The sample size of the first test sample n_2 : int or None (default=None) The sample size of the second test sample. If None , n_1 = n_2 . Returns z, p : float or None, float Returns the z-score and the p-value Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/proportion_difference/","title":"API"},{"location":"user_guide/evaluate/scoring/","text":"scoring: computing various performance metrics A function for computing various different performance metrics. from mlxtend.evaluate import scoring Overview Confusion Matrix The confusion matrix (or error matrix ) is one way to summarize the performance of a classifier for binary classification tasks. This square matrix consists of columns and rows that list the number of instances as absolute or relative \"actual class\" vs. \"predicted class\" ratios. Let P be the label of class 1 and N be the label of a second class or the label of all classes that are not class 1 in a multi-class setting. Error and Accuracy Both the prediction error (ERR) and accuracy (ACC) provide general information about how many samples are misclassified. The error can be understood as the sum of all false predictions divided by the number of total predications, and the the accuracy is calculated as the sum of correct predictions divided by the total number of predictions, respectively. ERR = \\frac{FP + FN}{FP+ FN + TP + TN} = 1-ACC ACC = \\frac{TP + TN}{FP+ FN + TP + TN} = 1-ERR True and False Positive Rates The True Positive Rate (TPR) and False Positive Rate (FPR) are performance metrics that are especially useful for imbalanced class problems. In spam classification , for example, we are of course primarily interested in the detection and filtering out of spam . However, it is also important to decrease the number of messages that were incorrectly classified as spam ( False Positives ): A situation where a person misses an important message is considered as \"worse\" than a situation where a person ends up with a few spam messages in his e-mail inbox. In contrast to the FPR , the True Positive Rate provides useful information about the fraction of positive (or relevant ) samples that were correctly identified out of the total pool of Positives . FPR = \\frac{FP}{N} = \\frac{FP}{FP + TN} TPR = \\frac{TP}{P} = \\frac{TP}{FN + TP} Precision, Recall, and the F1-Score Precision (PRE) and Recall (REC) are metrics that are more commonly used in Information Technology and related to the False and True Prositive Rates . In fact, Recall is synonymous to the True Positive Rate and also sometimes called Sensitivity . The F _1 -Score can be understood as a combination of both Precision and Recall . PRE = \\frac{TP}{TP + FP} REC = TPR = \\frac{TP}{P} = \\frac{TP}{FN + TP} F_1 = 2 \\cdot \\frac{PRE \\cdot REC}{PRE + REC} Sensitivity and Specificity Sensitivity (SEN) is synonymous to Recall and the True Positive Rate whereas Specificity (SPC) is synonymous to the True Negative Rate -- Sensitivity measures the recovery rate of the Positives and complimentary, the Specificity measures the recovery rate of the Negatives . SEN = TPR = REC = \\frac{TP}{P} = \\frac{TP}{FN + TP} SPC = TNR =\\frac{TN}{N} = \\frac{TN}{FP + TN} Matthews Correlation Coefficient Matthews correlation coefficient (MCC) was first formulated by Brian W. Matthews [3] in 1975 to assess the performance of protein secondary structure predictions. The MCC can be understood as a specific case of a linear correlation coefficient ( Pearson's R ) for a binary classification setting and is considered as especially useful in unbalanced class settings. The previous metrics take values in the range between 0 (worst) and 1 (best), whereas the MCC is bounded between the range 1 (perfect correlation between ground truth and predicted outcome) and -1 (inverse or negative correlation) -- a value of 0 denotes a random prediction. MCC = \\frac{ TP \\times TN - FP \\times FN } {\\sqrt{ (TP + FP) ( TP + FN ) ( TN + FP ) ( TN + FN ) } } Average Per-Class Accuracy The \"overall\" accuracy is defined as the number of correct predictions ( true positives TP and true negatives TN) over all samples n : ACC = \\frac{TP + TN}{n} in a binary class setting: In a multi-class setting, we can generalize the computation of the accuracy as the fraction of all true predictions (the diagonal) over all samples n. ACC = \\frac{T}{n} Considering a multi-class problem with 3 classes (C0, C1, C2) let's assume our model made the following predictions: We compute the accuracy as: ACC = \\frac{3 + 50 + 18}{90} \\approx 0.79 Now, in order to compute the average per-class accuracy , we compute the binary accuracy for each class label separately; i.e., if class 1 is the positive class, class 0 and 2 are both considered the negative class. APC\\;ACC = \\frac{83/90 + 71/90 + 78/90}{3} \\approx 0.86 References [1] S. Raschka. An overview of general performance metrics of binary classifier systems . Computing Research Repository (CoRR), abs/1410.5330, 2014. [2] Cyril Goutte and Eric Gaussier. A probabilistic interpretation of precision, recall and f-score, with implication for evaluation . In Advances in Information Retrieval, pages 345\u2013359. Springer, 2005. [3] Brian W Matthews. Comparison of the predicted and observed secondary structure of T4 phage lysozyme . Biochimica et Biophysica Acta (BBA)- Protein Structure, 405(2):442\u2013451, 1975. Example 1 - Classification Error from mlxtend.evaluate import scoring y_targ = [1, 1, 1, 0, 0, 2, 0, 3] y_pred = [1, 0, 1, 0, 0, 2, 1, 3] res = scoring(y_target=y_targ, y_predicted=y_pred, metric='error') print('Error: %s%%' % (res * 100)) Error: 25.0% API scoring(y_target, y_predicted, metric='error', positive_label=1, unique_labels='auto') Compute a scoring metric for supervised learning. Parameters y_target : array-like, shape=[n_values] True class labels or target values. y_predicted : array-like, shape=[n_values] Predicted class labels or target values. metric : str (default: 'error') Performance metric: 'accuracy': (TP + TN)/(FP + FN + TP + TN) = 1-ERR 'average per-class accuracy': Average per-class accuracy 'average per-class error': Average per-class error 'error': (TP + TN)/(FP+ FN + TP + TN) = 1-ACC 'false_positive_rate': FP/N = FP/(FP + TN) 'true_positive_rate': TP/P = TP/(FN + TP) 'true_negative_rate': TN/N = TN/(FP + TN) 'precision': TP/(TP + FP) 'recall': equal to 'true_positive_rate' 'sensitivity': equal to 'true_positive_rate' or 'recall' 'specificity': equal to 'true_negative_rate' 'f1': 2 * (PRE * REC)/(PRE + REC) 'matthews_corr_coef': (TP TN - FP FN) / (sqrt{(TP + FP)( TP + FN )( TN + FP )( TN + FN )}) Where: [TP: True positives, TN = True negatives, TN: True negatives, FN = False negatives] positive_label : int (default: 1) Label of the positive class for binary classification metrics. unique_labels : str or array-like (default: 'auto') If 'auto', deduces the unique class labels from y_target Returns score : float Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/scoring/","title":"scoring: computing various performance metrics"},{"location":"user_guide/evaluate/scoring/#scoring-computing-various-performance-metrics","text":"A function for computing various different performance metrics. from mlxtend.evaluate import scoring","title":"scoring: computing various performance metrics"},{"location":"user_guide/evaluate/scoring/#overview","text":"","title":"Overview"},{"location":"user_guide/evaluate/scoring/#confusion-matrix","text":"The confusion matrix (or error matrix ) is one way to summarize the performance of a classifier for binary classification tasks. This square matrix consists of columns and rows that list the number of instances as absolute or relative \"actual class\" vs. \"predicted class\" ratios. Let P be the label of class 1 and N be the label of a second class or the label of all classes that are not class 1 in a multi-class setting.","title":"Confusion Matrix"},{"location":"user_guide/evaluate/scoring/#error-and-accuracy","text":"Both the prediction error (ERR) and accuracy (ACC) provide general information about how many samples are misclassified. The error can be understood as the sum of all false predictions divided by the number of total predications, and the the accuracy is calculated as the sum of correct predictions divided by the total number of predictions, respectively. ERR = \\frac{FP + FN}{FP+ FN + TP + TN} = 1-ACC ACC = \\frac{TP + TN}{FP+ FN + TP + TN} = 1-ERR","title":"Error and Accuracy"},{"location":"user_guide/evaluate/scoring/#true-and-false-positive-rates","text":"The True Positive Rate (TPR) and False Positive Rate (FPR) are performance metrics that are especially useful for imbalanced class problems. In spam classification , for example, we are of course primarily interested in the detection and filtering out of spam . However, it is also important to decrease the number of messages that were incorrectly classified as spam ( False Positives ): A situation where a person misses an important message is considered as \"worse\" than a situation where a person ends up with a few spam messages in his e-mail inbox. In contrast to the FPR , the True Positive Rate provides useful information about the fraction of positive (or relevant ) samples that were correctly identified out of the total pool of Positives . FPR = \\frac{FP}{N} = \\frac{FP}{FP + TN} TPR = \\frac{TP}{P} = \\frac{TP}{FN + TP}","title":"True and False Positive Rates"},{"location":"user_guide/evaluate/scoring/#precision-recall-and-the-f1-score","text":"Precision (PRE) and Recall (REC) are metrics that are more commonly used in Information Technology and related to the False and True Prositive Rates . In fact, Recall is synonymous to the True Positive Rate and also sometimes called Sensitivity . The F _1 -Score can be understood as a combination of both Precision and Recall . PRE = \\frac{TP}{TP + FP} REC = TPR = \\frac{TP}{P} = \\frac{TP}{FN + TP} F_1 = 2 \\cdot \\frac{PRE \\cdot REC}{PRE + REC}","title":"Precision, Recall, and the F1-Score"},{"location":"user_guide/evaluate/scoring/#sensitivity-and-specificity","text":"Sensitivity (SEN) is synonymous to Recall and the True Positive Rate whereas Specificity (SPC) is synonymous to the True Negative Rate -- Sensitivity measures the recovery rate of the Positives and complimentary, the Specificity measures the recovery rate of the Negatives . SEN = TPR = REC = \\frac{TP}{P} = \\frac{TP}{FN + TP} SPC = TNR =\\frac{TN}{N} = \\frac{TN}{FP + TN}","title":"Sensitivity and Specificity"},{"location":"user_guide/evaluate/scoring/#matthews-correlation-coefficient","text":"Matthews correlation coefficient (MCC) was first formulated by Brian W. Matthews [3] in 1975 to assess the performance of protein secondary structure predictions. The MCC can be understood as a specific case of a linear correlation coefficient ( Pearson's R ) for a binary classification setting and is considered as especially useful in unbalanced class settings. The previous metrics take values in the range between 0 (worst) and 1 (best), whereas the MCC is bounded between the range 1 (perfect correlation between ground truth and predicted outcome) and -1 (inverse or negative correlation) -- a value of 0 denotes a random prediction. MCC = \\frac{ TP \\times TN - FP \\times FN } {\\sqrt{ (TP + FP) ( TP + FN ) ( TN + FP ) ( TN + FN ) } }","title":"Matthews Correlation Coefficient"},{"location":"user_guide/evaluate/scoring/#average-per-class-accuracy","text":"The \"overall\" accuracy is defined as the number of correct predictions ( true positives TP and true negatives TN) over all samples n : ACC = \\frac{TP + TN}{n} in a binary class setting: In a multi-class setting, we can generalize the computation of the accuracy as the fraction of all true predictions (the diagonal) over all samples n. ACC = \\frac{T}{n} Considering a multi-class problem with 3 classes (C0, C1, C2) let's assume our model made the following predictions: We compute the accuracy as: ACC = \\frac{3 + 50 + 18}{90} \\approx 0.79 Now, in order to compute the average per-class accuracy , we compute the binary accuracy for each class label separately; i.e., if class 1 is the positive class, class 0 and 2 are both considered the negative class. APC\\;ACC = \\frac{83/90 + 71/90 + 78/90}{3} \\approx 0.86","title":"Average Per-Class Accuracy"},{"location":"user_guide/evaluate/scoring/#references","text":"[1] S. Raschka. An overview of general performance metrics of binary classifier systems . Computing Research Repository (CoRR), abs/1410.5330, 2014. [2] Cyril Goutte and Eric Gaussier. A probabilistic interpretation of precision, recall and f-score, with implication for evaluation . In Advances in Information Retrieval, pages 345\u2013359. Springer, 2005. [3] Brian W Matthews. Comparison of the predicted and observed secondary structure of T4 phage lysozyme . Biochimica et Biophysica Acta (BBA)- Protein Structure, 405(2):442\u2013451, 1975.","title":"References"},{"location":"user_guide/evaluate/scoring/#example-1-classification-error","text":"from mlxtend.evaluate import scoring y_targ = [1, 1, 1, 0, 0, 2, 0, 3] y_pred = [1, 0, 1, 0, 0, 2, 1, 3] res = scoring(y_target=y_targ, y_predicted=y_pred, metric='error') print('Error: %s%%' % (res * 100)) Error: 25.0%","title":"Example 1 - Classification Error"},{"location":"user_guide/evaluate/scoring/#api","text":"scoring(y_target, y_predicted, metric='error', positive_label=1, unique_labels='auto') Compute a scoring metric for supervised learning. Parameters y_target : array-like, shape=[n_values] True class labels or target values. y_predicted : array-like, shape=[n_values] Predicted class labels or target values. metric : str (default: 'error') Performance metric: 'accuracy': (TP + TN)/(FP + FN + TP + TN) = 1-ERR 'average per-class accuracy': Average per-class accuracy 'average per-class error': Average per-class error 'error': (TP + TN)/(FP+ FN + TP + TN) = 1-ACC 'false_positive_rate': FP/N = FP/(FP + TN) 'true_positive_rate': TP/P = TP/(FN + TP) 'true_negative_rate': TN/N = TN/(FP + TN) 'precision': TP/(TP + FP) 'recall': equal to 'true_positive_rate' 'sensitivity': equal to 'true_positive_rate' or 'recall' 'specificity': equal to 'true_negative_rate' 'f1': 2 * (PRE * REC)/(PRE + REC) 'matthews_corr_coef': (TP TN - FP FN) / (sqrt{(TP + FP)( TP + FN )( TN + FP )( TN + FN )}) Where: [TP: True positives, TN = True negatives, TN: True negatives, FN = False negatives] positive_label : int (default: 1) Label of the positive class for binary classification metrics. unique_labels : str or array-like (default: 'auto') If 'auto', deduces the unique class labels from y_target Returns score : float Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/evaluate/scoring/","title":"API"},{"location":"user_guide/feature_extraction/LinearDiscriminantAnalysis/","text":"LinearDiscriminantAnalysis: Linear discriminant analysis for dimensionality reduction Implementation of Linear Discriminant Analysis for dimensionality reduction from mlxtend.feature_extraction import LinearDiscriminantAnalysis Overview Linear Discriminant Analysis (LDA) is most commonly used as dimensionality reduction technique in the pre-processing step for pattern-classification and machine learning applications. The goal is to project a dataset onto a lower-dimensional space with good class-separability in order avoid overfitting (\"curse of dimensionality\") and also reduce computational costs. Ronald A. Fisher formulated the Linear Discriminant in 1936 ( The Use of Multiple Measurements in Taxonomic Problems ), and it also has some practical uses as classifier. The original Linear discriminant was described for a 2-class problem, and it was then later generalized as \"multi-class Linear Discriminant Analysis\" or \"Multiple Discriminant Analysis\" by C. R. Rao in 1948 ( The utilization of multiple measurements in problems of biological classification ) The general LDA approach is very similar to a Principal Component Analysis, but in addition to finding the component axes that maximize the variance of our data (PCA), we are additionally interested in the axes that maximize the separation between multiple classes (LDA). So, in a nutshell, often the goal of an LDA is to project a feature space (a dataset n-dimensional samples) onto a smaller subspace k (where k \\leq n-1 ) while maintaining the class-discriminatory information. In general, dimensionality reduction does not only help reducing computational costs for a given classification task, but it can also be helpful to avoid overfitting by minimizing the error in parameter estimation (\"curse of dimensionality\"). Summarizing the LDA approach in 5 steps Listed below are the 5 general steps for performing a linear discriminant analysis. Compute the d -dimensional mean vectors for the different classes from the dataset. Compute the scatter matrices (in-between-class and within-class scatter matrix). Compute the eigenvectors ( \\mathbf{e_1}, \\; \\mathbf{e_2}, \\; ..., \\; \\mathbf{e_d} ) and corresponding eigenvalues ( \\mathbf{\\lambda_1}, \\; \\mathbf{\\lambda_2}, \\; ..., \\; \\mathbf{\\lambda_d} ) for the scatter matrices. Sort the eigenvectors by decreasing eigenvalues and choose k eigenvectors with the largest eigenvalues to form a k \\times d dimensional matrix \\mathbf{W} (where every column represents an eigenvector). Use this k \\times d eigenvector matrix to transform the samples onto the new subspace. This can be summarized by the mathematical equation: \\mathbf{Y} = \\mathbf{X} \\times \\mathbf{W} (where \\mathbf{X} is a n \\times d -dimensional matrix representing the n samples, and \\mathbf{y} are the transformed n \\times k -dimensional samples in the new subspace). References Fisher, Ronald A. \" The use of multiple measurements in taxonomic problems. \" Annals of eugenics 7.2 (1936): 179-188. Rao, C. Radhakrishna. \" The utilization of multiple measurements in problems of biological classification. \" Journal of the Royal Statistical Society. Series B (Methodological) 10.2 (1948): 159-203. Example 1 - LDA on Iris from mlxtend.data import iris_data from mlxtend.preprocessing import standardize from mlxtend.feature_extraction import LinearDiscriminantAnalysis X, y = iris_data() X = standardize(X) lda = LinearDiscriminantAnalysis(n_discriminants=2) lda.fit(X, y) X_lda = lda.transform(X) import matplotlib.pyplot as plt with plt.style.context('seaborn-whitegrid'): plt.figure(figsize=(6, 4)) for lab, col in zip((0, 1, 2), ('blue', 'red', 'green')): plt.scatter(X_lda[y == lab, 0], X_lda[y == lab, 1], label=lab, c=col) plt.xlabel('Linear Discriminant 1') plt.ylabel('Linear Discriminant 2') plt.legend(loc='lower right') plt.tight_layout() plt.show() Example 2 - Plotting the Between-Class Variance Explained Ratio from mlxtend.data import iris_data from mlxtend.preprocessing import standardize from mlxtend.feature_extraction import LinearDiscriminantAnalysis X, y = iris_data() X = standardize(X) lda = LinearDiscriminantAnalysis(n_discriminants=None) lda.fit(X, y) X_lda = lda.transform(X) import numpy as np tot = sum(lda.e_vals_) var_exp = [(i / tot)*100 for i in sorted(lda.e_vals_, reverse=True)] cum_var_exp = np.cumsum(var_exp) with plt.style.context('seaborn-whitegrid'): fig, ax = plt.subplots(figsize=(6, 4)) plt.bar(range(4), var_exp, alpha=0.5, align='center', label='individual explained variance') plt.step(range(4), cum_var_exp, where='mid', label='cumulative explained variance') plt.ylabel('Explained variance ratio') plt.xlabel('Principal components') plt.xticks(range(4)) ax.set_xticklabels(np.arange(1, X.shape[1] + 1)) plt.legend(loc='best') plt.tight_layout() API LinearDiscriminantAnalysis(n_discriminants=None) Linear Discriminant Analysis Class Parameters n_discriminants : int (default: None) The number of discrimants for transformation. Keeps the original dimensions of the dataset if None . Attributes w_ : array-like, shape=[n_features, n_discriminants] Projection matrix e_vals_ : array-like, shape=[n_features] Eigenvalues in sorted order. e_vecs_ : array-like, shape=[n_features] Eigenvectors in sorted order. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/feature_extraction/LinearDiscriminantAnalysis/ Methods fit(X, y, n_classes=None) Fit the LDA model with X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values. n_classes : int (default: None) A positive integer to declare the number of class labels if not all class labels are present in a partial training set. Gets the number of class labels automatically if None. Returns self : object transform(X) Apply the linear transformation on X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns X_projected : np.ndarray, shape = [n_samples, n_discriminants] Projected training vectors.","title":"LinearDiscriminantAnalysis: Linear discriminant analysis for dimensionality reduction"},{"location":"user_guide/feature_extraction/LinearDiscriminantAnalysis/#lineardiscriminantanalysis-linear-discriminant-analysis-for-dimensionality-reduction","text":"Implementation of Linear Discriminant Analysis for dimensionality reduction from mlxtend.feature_extraction import LinearDiscriminantAnalysis","title":"LinearDiscriminantAnalysis: Linear discriminant analysis for dimensionality reduction"},{"location":"user_guide/feature_extraction/LinearDiscriminantAnalysis/#overview","text":"Linear Discriminant Analysis (LDA) is most commonly used as dimensionality reduction technique in the pre-processing step for pattern-classification and machine learning applications. The goal is to project a dataset onto a lower-dimensional space with good class-separability in order avoid overfitting (\"curse of dimensionality\") and also reduce computational costs. Ronald A. Fisher formulated the Linear Discriminant in 1936 ( The Use of Multiple Measurements in Taxonomic Problems ), and it also has some practical uses as classifier. The original Linear discriminant was described for a 2-class problem, and it was then later generalized as \"multi-class Linear Discriminant Analysis\" or \"Multiple Discriminant Analysis\" by C. R. Rao in 1948 ( The utilization of multiple measurements in problems of biological classification ) The general LDA approach is very similar to a Principal Component Analysis, but in addition to finding the component axes that maximize the variance of our data (PCA), we are additionally interested in the axes that maximize the separation between multiple classes (LDA). So, in a nutshell, often the goal of an LDA is to project a feature space (a dataset n-dimensional samples) onto a smaller subspace k (where k \\leq n-1 ) while maintaining the class-discriminatory information. In general, dimensionality reduction does not only help reducing computational costs for a given classification task, but it can also be helpful to avoid overfitting by minimizing the error in parameter estimation (\"curse of dimensionality\").","title":"Overview"},{"location":"user_guide/feature_extraction/LinearDiscriminantAnalysis/#summarizing-the-lda-approach-in-5-steps","text":"Listed below are the 5 general steps for performing a linear discriminant analysis. Compute the d -dimensional mean vectors for the different classes from the dataset. Compute the scatter matrices (in-between-class and within-class scatter matrix). Compute the eigenvectors ( \\mathbf{e_1}, \\; \\mathbf{e_2}, \\; ..., \\; \\mathbf{e_d} ) and corresponding eigenvalues ( \\mathbf{\\lambda_1}, \\; \\mathbf{\\lambda_2}, \\; ..., \\; \\mathbf{\\lambda_d} ) for the scatter matrices. Sort the eigenvectors by decreasing eigenvalues and choose k eigenvectors with the largest eigenvalues to form a k \\times d dimensional matrix \\mathbf{W} (where every column represents an eigenvector). Use this k \\times d eigenvector matrix to transform the samples onto the new subspace. This can be summarized by the mathematical equation: \\mathbf{Y} = \\mathbf{X} \\times \\mathbf{W} (where \\mathbf{X} is a n \\times d -dimensional matrix representing the n samples, and \\mathbf{y} are the transformed n \\times k -dimensional samples in the new subspace).","title":"Summarizing the LDA approach in 5 steps"},{"location":"user_guide/feature_extraction/LinearDiscriminantAnalysis/#references","text":"Fisher, Ronald A. \" The use of multiple measurements in taxonomic problems. \" Annals of eugenics 7.2 (1936): 179-188. Rao, C. Radhakrishna. \" The utilization of multiple measurements in problems of biological classification. \" Journal of the Royal Statistical Society. Series B (Methodological) 10.2 (1948): 159-203.","title":"References"},{"location":"user_guide/feature_extraction/LinearDiscriminantAnalysis/#example-1-lda-on-iris","text":"from mlxtend.data import iris_data from mlxtend.preprocessing import standardize from mlxtend.feature_extraction import LinearDiscriminantAnalysis X, y = iris_data() X = standardize(X) lda = LinearDiscriminantAnalysis(n_discriminants=2) lda.fit(X, y) X_lda = lda.transform(X) import matplotlib.pyplot as plt with plt.style.context('seaborn-whitegrid'): plt.figure(figsize=(6, 4)) for lab, col in zip((0, 1, 2), ('blue', 'red', 'green')): plt.scatter(X_lda[y == lab, 0], X_lda[y == lab, 1], label=lab, c=col) plt.xlabel('Linear Discriminant 1') plt.ylabel('Linear Discriminant 2') plt.legend(loc='lower right') plt.tight_layout() plt.show()","title":"Example 1 - LDA on Iris"},{"location":"user_guide/feature_extraction/LinearDiscriminantAnalysis/#example-2-plotting-the-between-class-variance-explained-ratio","text":"from mlxtend.data import iris_data from mlxtend.preprocessing import standardize from mlxtend.feature_extraction import LinearDiscriminantAnalysis X, y = iris_data() X = standardize(X) lda = LinearDiscriminantAnalysis(n_discriminants=None) lda.fit(X, y) X_lda = lda.transform(X) import numpy as np tot = sum(lda.e_vals_) var_exp = [(i / tot)*100 for i in sorted(lda.e_vals_, reverse=True)] cum_var_exp = np.cumsum(var_exp) with plt.style.context('seaborn-whitegrid'): fig, ax = plt.subplots(figsize=(6, 4)) plt.bar(range(4), var_exp, alpha=0.5, align='center', label='individual explained variance') plt.step(range(4), cum_var_exp, where='mid', label='cumulative explained variance') plt.ylabel('Explained variance ratio') plt.xlabel('Principal components') plt.xticks(range(4)) ax.set_xticklabels(np.arange(1, X.shape[1] + 1)) plt.legend(loc='best') plt.tight_layout()","title":"Example 2 - Plotting the Between-Class Variance Explained Ratio"},{"location":"user_guide/feature_extraction/LinearDiscriminantAnalysis/#api","text":"LinearDiscriminantAnalysis(n_discriminants=None) Linear Discriminant Analysis Class Parameters n_discriminants : int (default: None) The number of discrimants for transformation. Keeps the original dimensions of the dataset if None . Attributes w_ : array-like, shape=[n_features, n_discriminants] Projection matrix e_vals_ : array-like, shape=[n_features] Eigenvalues in sorted order. e_vecs_ : array-like, shape=[n_features] Eigenvectors in sorted order. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/feature_extraction/LinearDiscriminantAnalysis/","title":"API"},{"location":"user_guide/feature_extraction/LinearDiscriminantAnalysis/#methods","text":"fit(X, y, n_classes=None) Fit the LDA model with X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values. n_classes : int (default: None) A positive integer to declare the number of class labels if not all class labels are present in a partial training set. Gets the number of class labels automatically if None. Returns self : object transform(X) Apply the linear transformation on X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns X_projected : np.ndarray, shape = [n_samples, n_discriminants] Projected training vectors.","title":"Methods"},{"location":"user_guide/feature_extraction/PrincipalComponentAnalysis/","text":"PrincipalComponentAnalysis: Principal component analysis (PCA) for dimensionality reduction Implementation of Principal Component Analysis for dimensionality reduction from mlxtend.feature_extraction import PrincipalComponentAnalysis Overview The sheer size of data in the modern age is not only a challenge for computer hardware but also a main bottleneck for the performance of many machine learning algorithms. The main goal of a PCA analysis is to identify patterns in data; PCA aims to detect the correlation between variables. If a strong correlation between variables exists, the attempt to reduce the dimensionality only makes sense. In a nutshell, this is what PCA is all about: Finding the directions of maximum variance in high-dimensional data and project it onto a smaller dimensional subspace while retaining most of the information. PCA and Dimensionality Reduction Often, the desired goal is to reduce the dimensions of a d -dimensional dataset by projecting it onto a (k) -dimensional subspace (where k\\;<\\;d ) in order to increase the computational efficiency while retaining most of the information. An important question is \"what is the size of k that represents the data 'well'?\" Later, we will compute eigenvectors (the principal components) of a dataset and collect them in a projection matrix. Each of those eigenvectors is associated with an eigenvalue which can be interpreted as the \"length\" or \"magnitude\" of the corresponding eigenvector. If some eigenvalues have a significantly larger magnitude than others that the reduction of the dataset via PCA onto a smaller dimensional subspace by dropping the \"less informative\" eigenpairs is reasonable. A Summary of the PCA Approach Standardize the data. Obtain the Eigenvectors and Eigenvalues from the covariance matrix or correlation matrix, or perform Singular Vector Decomposition. Sort eigenvalues in descending order and choose the k eigenvectors that correspond to the k largest eigenvalues where k is the number of dimensions of the new feature subspace ( k \\le d ). Construct the projection matrix \\mathbf{W} from the selected k eigenvectors. Transform the original dataset \\mathbf{X} via \\mathbf{W} to obtain a k -dimensional feature subspace \\mathbf{Y} . References Pearson, Karl. \"LIII. On lines and planes of closest fit to systems of points in space. \" The London, Edinburgh, and Dublin Philosophical Magazine and Journal of Science 2.11 (1901): 559-572. Example 1 - PCA on Iris from mlxtend.data import iris_data from mlxtend.preprocessing import standardize from mlxtend.feature_extraction import PrincipalComponentAnalysis X, y = iris_data() X = standardize(X) pca = PrincipalComponentAnalysis(n_components=2) pca.fit(X) X_pca = pca.transform(X) import matplotlib.pyplot as plt with plt.style.context('seaborn-whitegrid'): plt.figure(figsize=(6, 4)) for lab, col in zip((0, 1, 2), ('blue', 'red', 'green')): plt.scatter(X_pca[y==lab, 0], X_pca[y==lab, 1], label=lab, c=col) plt.xlabel('Principal Component 1') plt.ylabel('Principal Component 2') plt.legend(loc='lower center') plt.tight_layout() plt.show() Example 2 - Plotting the Variance Explained Ratio from mlxtend.data import iris_data from mlxtend.preprocessing import standardize X, y = iris_data() X = standardize(X) pca = PrincipalComponentAnalysis(n_components=None) pca.fit(X) X_pca = pca.transform(X) pca.e_vals_ array([2.91081808, 0.92122093, 0.14735328, 0.02060771]) pca.e_vals_normalized_ array([0.72770452, 0.23030523, 0.03683832, 0.00515193]) import numpy as np tot = sum(pca.e_vals_) var_exp = [(i / tot)*100 for i in sorted(pca.e_vals_, reverse=True)] cum_var_exp = np.cumsum(pca.e_vals_normalized_*100) with plt.style.context('seaborn-whitegrid'): fig, ax = plt.subplots(figsize=(6, 4)) plt.bar(range(4), var_exp, alpha=0.5, align='center', label='individual explained variance') plt.step(range(4), cum_var_exp, where='mid', label='cumulative explained variance') plt.ylabel('Explained variance ratio') plt.xlabel('Principal components') plt.xticks(range(4)) ax.set_xticklabels(np.arange(1, X.shape[1] + 1)) plt.legend(loc='best') plt.tight_layout() Example 3 - PCA via SVD While the eigendecomposition of the covariance or correlation matrix may be more intuitiuve, most PCA implementations perform a Singular Vector Decomposition (SVD) to improve the computational efficiency. Another advantage of using SVD is that the results tend to be more numerically stable, since we can decompose the input matrix directly without the additional covariance-matrix step. from mlxtend.data import iris_data from mlxtend.preprocessing import standardize from mlxtend.feature_extraction import PrincipalComponentAnalysis X, y = iris_data() X = standardize(X) pca = PrincipalComponentAnalysis(n_components=2, solver='svd') pca.fit(X) X_pca = pca.transform(X) import matplotlib.pyplot as plt with plt.style.context('seaborn-whitegrid'): plt.figure(figsize=(6, 4)) for lab, col in zip((0, 1, 2), ('blue', 'red', 'green')): plt.scatter(X_pca[y==lab, 0], X_pca[y==lab, 1], label=lab, c=col) plt.xlabel('Principal Component 1') plt.ylabel('Principal Component 2') plt.legend(loc='lower center') plt.tight_layout() plt.show() If we compare this PCA projection to the previous plot in example 1, we notice that they are mirror images of each other. Note that this is not due to an error in any of those two implementations, but the reason for this difference is that, depending on the eigensolver, eigenvectors can have either negative or positive signs. For instance, if v is an eigenvector of a matrix \\Sigma , we have \\Sigma v = \\lambda v, where \\lambda is our eigenvalue then -v is also an eigenvector that has the same eigenvalue, since \\Sigma(-v) = -\\Sigma v = -\\lambda v = \\lambda(-v). Example 4 - Factor Loadings After evoking the fit method, the factor loadings are available via the loadings_ attribute. In simple terms, the loadings are the unstandardized values of the eigenvectors. Or in other words, we can interpret the loadings as the covariances (or correlation in case we standardized the input features) between the input features and the principal components (or eigenvectors), which have been scaled to unit length. By having the loadings scaled, they become comparable by magnitude and we can assess how much variance in a component is attributed to the input features (as the components are just a weighted linear combination of the input features). from mlxtend.data import iris_data from mlxtend.preprocessing import standardize from mlxtend.feature_extraction import PrincipalComponentAnalysis import matplotlib.pyplot as plt X, y = iris_data() X = standardize(X) pca = PrincipalComponentAnalysis(n_components=2, solver='eigen') pca.fit(X); xlabels = ['sepal length', 'sepal width', 'petal length', 'petal width'] fig, ax = plt.subplots(1, 2, figsize=(8, 3)) ax[0].bar(range(4), pca.loadings_[:, 0], align='center') ax[1].bar(range(4), pca.loadings_[:, 1], align='center') ax[0].set_ylabel('Factor loading onto PC1') ax[1].set_ylabel('Factor loading onto PC2') ax[0].set_xticks(range(4)) ax[1].set_xticks(range(4)) ax[0].set_xticklabels(xlabels, rotation=45) ax[1].set_xticklabels(xlabels, rotation=45) plt.ylim([-1, 1]) plt.tight_layout() For instance, we may say that most of the variance in the first component is attributed to the petal features (although the loading of sepal length on PC1 is also not much less in magnitude). In contrast, the remaining variance captured by PC2 is mostly due to the sepal width. Note that we know from Example 2 that PC1 explains most of the variance, and based on the information from the loading plots, we may say that petal features combined with sepal length may explain most of the spread in the data. Example 5 - Feature Extraction Pipeline from sklearn.pipeline import make_pipeline from sklearn.neighbors import KNeighborsClassifier from sklearn.preprocessing import StandardScaler from sklearn.model_selection import train_test_split from mlxtend.data import wine_data X, y = wine_data() X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=123, test_size=0.3, stratify=y) pipe_pca = make_pipeline(StandardScaler(), PrincipalComponentAnalysis(n_components=3), KNeighborsClassifier(n_neighbors=5)) pipe_pca.fit(X_train, y_train) print('Transf. training accyracy: %.2f%%' % (pipe_pca.score(X_train, y_train)*100)) print('Transf. test accyracy: %.2f%%' % (pipe_pca.score(X_test, y_test)*100)) Transf. training accyracy: 96.77% Transf. test accyracy: 96.30% Example 6 - Whitening Certain algorithms require the data to be whitened. This means that the features have unit variance and the off-diagonals are all zero (i.e., the features are uncorrelated). PCA already ensures that the features are uncorrelated, hence, we only need to apply a simple scaling to whiten the transformed data. For instance, for a given transformed feature X'_i , we divide it by the square-root of the corresponding eigenvalue \\lambda_i : X'_{\\text{whitened}} = \\frac{X'_i}{\\sqrt{\\lambda_i}}. The whitening via the PrincipalComponentAnalysis can be achieved by setting whitening=True during initialization. Let's demonstrate that with an example. from sklearn.preprocessing import StandardScaler from sklearn.model_selection import train_test_split from mlxtend.data import wine_data X, y = wine_data() X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=123, test_size=0.3, stratify=y) Regular PCA sc = StandardScaler() pca1 = PrincipalComponentAnalysis(n_components=2) X_train_scaled = sc.fit_transform(X_train) X_train_transf = pca1.fit(X_train_scaled).transform(X_train_scaled) with plt.style.context('seaborn-whitegrid'): plt.figure(figsize=(6, 4)) for lab, col in zip((0, 1, 2), ('blue', 'red', 'green')): plt.scatter(X_train_transf[y_train==lab, 0], X_train_transf[y_train==lab, 1], label=lab, c=col) plt.xlabel('Principal Component 1') plt.ylabel('Principal Component 2') plt.legend(loc='lower center') plt.tight_layout() plt.show() np.set_printoptions(precision=1, suppress=True) print('Covariance matrix:\\n') np.cov(X_train_transf.T) Covariance matrix: array([[4.9, 0. ], [0. , 2.5]]) As we can see, the features are uncorrelated after transformation but don't have unit variance. PCA with Whitening sc = StandardScaler() pca1 = PrincipalComponentAnalysis(n_components=2, whitening=True) X_train_scaled = sc.fit_transform(X_train) X_train_transf = pca1.fit(X_train_scaled).transform(X_train_scaled) with plt.style.context('seaborn-whitegrid'): plt.figure(figsize=(6, 4)) for lab, col in zip((0, 1, 2), ('blue', 'red', 'green')): plt.scatter(X_train_transf[y_train==lab, 0], X_train_transf[y_train==lab, 1], label=lab, c=col) plt.xlabel('Principal Component 1') plt.ylabel('Principal Component 2') plt.legend(loc='lower center') plt.tight_layout() plt.show() np.set_printoptions(precision=1, suppress=True) print('Covariance matrix:\\n') np.cov(X_train_transf.T) Covariance matrix: array([[1., 0.], [0., 1.]]) As we can see above, the whitening achieves that all features now have unit variance. I.e., the covariance matrix of the transformed features becomes the identity matrix. API PrincipalComponentAnalysis(n_components=None, solver='svd', whitening=False) Principal Component Analysis Class Parameters n_components : int (default: None) The number of principal components for transformation. Keeps the original dimensions of the dataset if None . solver : str (default: 'svd') Method for performing the matrix decomposition. {'eigen', 'svd'} whitening : bool (default: False) Performs whitening such that the covariance matrix of the transformed data will be the identity matrix. Attributes w_ : array-like, shape=[n_features, n_components] Projection matrix e_vals_ : array-like, shape=[n_features] Eigenvalues in sorted order. e_vecs_ : array-like, shape=[n_features] Eigenvectors in sorted order. e_vals_normalized_ : array-like, shape=[n_features] Normalized eigen values such that they sum up to 1. This is equal to what's often referred to as \"explained variance ratios.\" loadings_ : array_like, shape=[n_features, n_features] The factor loadings of the original variables onto the principal components. The columns are the principal components, and the rows are the features loadings. For instance, the first column contains the loadings onto the first principal component. Note that the signs may be flipped depending on whether you use the 'eigen' or 'svd' solver; this does not affect the interpretation of the loadings though. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/feature_extraction/PrincipalComponentAnalysis/ Methods fit(X, y=None) Learn model from training data. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns self : object get_params(deep=True) Get parameters for this estimator. Parameters deep : boolean, optional If True, will return the parameters for this estimator and contained subobjects that are estimators. Returns params : mapping of string to any Parameter names mapped to their values.' adapted from https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/base.py Author: Gael Varoquaux gael.varoquaux@normalesup.org License: BSD 3 clause set_params( params) Set the parameters of this estimator. The method works on simple estimators as well as on nested objects (such as pipelines). The latter have parameters of the form <component>__<parameter> so that it's possible to update each component of a nested object. Returns self adapted from https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/base.py Author: Gael Varoquaux gael.varoquaux@normalesup.org License: BSD 3 clause transform(X) Apply the linear transformation on X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns X_projected : np.ndarray, shape = [n_samples, n_components] Projected training vectors.","title":"PrincipalComponentAnalysis: Principal component analysis (PCA) for dimensionality reduction"},{"location":"user_guide/feature_extraction/PrincipalComponentAnalysis/#principalcomponentanalysis-principal-component-analysis-pca-for-dimensionality-reduction","text":"Implementation of Principal Component Analysis for dimensionality reduction from mlxtend.feature_extraction import PrincipalComponentAnalysis","title":"PrincipalComponentAnalysis: Principal component analysis (PCA) for dimensionality reduction"},{"location":"user_guide/feature_extraction/PrincipalComponentAnalysis/#overview","text":"The sheer size of data in the modern age is not only a challenge for computer hardware but also a main bottleneck for the performance of many machine learning algorithms. The main goal of a PCA analysis is to identify patterns in data; PCA aims to detect the correlation between variables. If a strong correlation between variables exists, the attempt to reduce the dimensionality only makes sense. In a nutshell, this is what PCA is all about: Finding the directions of maximum variance in high-dimensional data and project it onto a smaller dimensional subspace while retaining most of the information.","title":"Overview"},{"location":"user_guide/feature_extraction/PrincipalComponentAnalysis/#pca-and-dimensionality-reduction","text":"Often, the desired goal is to reduce the dimensions of a d -dimensional dataset by projecting it onto a (k) -dimensional subspace (where k\\;<\\;d ) in order to increase the computational efficiency while retaining most of the information. An important question is \"what is the size of k that represents the data 'well'?\" Later, we will compute eigenvectors (the principal components) of a dataset and collect them in a projection matrix. Each of those eigenvectors is associated with an eigenvalue which can be interpreted as the \"length\" or \"magnitude\" of the corresponding eigenvector. If some eigenvalues have a significantly larger magnitude than others that the reduction of the dataset via PCA onto a smaller dimensional subspace by dropping the \"less informative\" eigenpairs is reasonable.","title":"PCA and Dimensionality Reduction"},{"location":"user_guide/feature_extraction/PrincipalComponentAnalysis/#a-summary-of-the-pca-approach","text":"Standardize the data. Obtain the Eigenvectors and Eigenvalues from the covariance matrix or correlation matrix, or perform Singular Vector Decomposition. Sort eigenvalues in descending order and choose the k eigenvectors that correspond to the k largest eigenvalues where k is the number of dimensions of the new feature subspace ( k \\le d ). Construct the projection matrix \\mathbf{W} from the selected k eigenvectors. Transform the original dataset \\mathbf{X} via \\mathbf{W} to obtain a k -dimensional feature subspace \\mathbf{Y} .","title":"A Summary of the PCA Approach"},{"location":"user_guide/feature_extraction/PrincipalComponentAnalysis/#references","text":"Pearson, Karl. \"LIII. On lines and planes of closest fit to systems of points in space. \" The London, Edinburgh, and Dublin Philosophical Magazine and Journal of Science 2.11 (1901): 559-572.","title":"References"},{"location":"user_guide/feature_extraction/PrincipalComponentAnalysis/#example-1-pca-on-iris","text":"from mlxtend.data import iris_data from mlxtend.preprocessing import standardize from mlxtend.feature_extraction import PrincipalComponentAnalysis X, y = iris_data() X = standardize(X) pca = PrincipalComponentAnalysis(n_components=2) pca.fit(X) X_pca = pca.transform(X) import matplotlib.pyplot as plt with plt.style.context('seaborn-whitegrid'): plt.figure(figsize=(6, 4)) for lab, col in zip((0, 1, 2), ('blue', 'red', 'green')): plt.scatter(X_pca[y==lab, 0], X_pca[y==lab, 1], label=lab, c=col) plt.xlabel('Principal Component 1') plt.ylabel('Principal Component 2') plt.legend(loc='lower center') plt.tight_layout() plt.show()","title":"Example 1 - PCA on Iris"},{"location":"user_guide/feature_extraction/PrincipalComponentAnalysis/#example-2-plotting-the-variance-explained-ratio","text":"from mlxtend.data import iris_data from mlxtend.preprocessing import standardize X, y = iris_data() X = standardize(X) pca = PrincipalComponentAnalysis(n_components=None) pca.fit(X) X_pca = pca.transform(X) pca.e_vals_ array([2.91081808, 0.92122093, 0.14735328, 0.02060771]) pca.e_vals_normalized_ array([0.72770452, 0.23030523, 0.03683832, 0.00515193]) import numpy as np tot = sum(pca.e_vals_) var_exp = [(i / tot)*100 for i in sorted(pca.e_vals_, reverse=True)] cum_var_exp = np.cumsum(pca.e_vals_normalized_*100) with plt.style.context('seaborn-whitegrid'): fig, ax = plt.subplots(figsize=(6, 4)) plt.bar(range(4), var_exp, alpha=0.5, align='center', label='individual explained variance') plt.step(range(4), cum_var_exp, where='mid', label='cumulative explained variance') plt.ylabel('Explained variance ratio') plt.xlabel('Principal components') plt.xticks(range(4)) ax.set_xticklabels(np.arange(1, X.shape[1] + 1)) plt.legend(loc='best') plt.tight_layout()","title":"Example 2 - Plotting the Variance Explained Ratio"},{"location":"user_guide/feature_extraction/PrincipalComponentAnalysis/#example-3-pca-via-svd","text":"While the eigendecomposition of the covariance or correlation matrix may be more intuitiuve, most PCA implementations perform a Singular Vector Decomposition (SVD) to improve the computational efficiency. Another advantage of using SVD is that the results tend to be more numerically stable, since we can decompose the input matrix directly without the additional covariance-matrix step. from mlxtend.data import iris_data from mlxtend.preprocessing import standardize from mlxtend.feature_extraction import PrincipalComponentAnalysis X, y = iris_data() X = standardize(X) pca = PrincipalComponentAnalysis(n_components=2, solver='svd') pca.fit(X) X_pca = pca.transform(X) import matplotlib.pyplot as plt with plt.style.context('seaborn-whitegrid'): plt.figure(figsize=(6, 4)) for lab, col in zip((0, 1, 2), ('blue', 'red', 'green')): plt.scatter(X_pca[y==lab, 0], X_pca[y==lab, 1], label=lab, c=col) plt.xlabel('Principal Component 1') plt.ylabel('Principal Component 2') plt.legend(loc='lower center') plt.tight_layout() plt.show() If we compare this PCA projection to the previous plot in example 1, we notice that they are mirror images of each other. Note that this is not due to an error in any of those two implementations, but the reason for this difference is that, depending on the eigensolver, eigenvectors can have either negative or positive signs. For instance, if v is an eigenvector of a matrix \\Sigma , we have \\Sigma v = \\lambda v, where \\lambda is our eigenvalue then -v is also an eigenvector that has the same eigenvalue, since \\Sigma(-v) = -\\Sigma v = -\\lambda v = \\lambda(-v).","title":"Example 3 - PCA via SVD"},{"location":"user_guide/feature_extraction/PrincipalComponentAnalysis/#example-4-factor-loadings","text":"After evoking the fit method, the factor loadings are available via the loadings_ attribute. In simple terms, the loadings are the unstandardized values of the eigenvectors. Or in other words, we can interpret the loadings as the covariances (or correlation in case we standardized the input features) between the input features and the principal components (or eigenvectors), which have been scaled to unit length. By having the loadings scaled, they become comparable by magnitude and we can assess how much variance in a component is attributed to the input features (as the components are just a weighted linear combination of the input features). from mlxtend.data import iris_data from mlxtend.preprocessing import standardize from mlxtend.feature_extraction import PrincipalComponentAnalysis import matplotlib.pyplot as plt X, y = iris_data() X = standardize(X) pca = PrincipalComponentAnalysis(n_components=2, solver='eigen') pca.fit(X); xlabels = ['sepal length', 'sepal width', 'petal length', 'petal width'] fig, ax = plt.subplots(1, 2, figsize=(8, 3)) ax[0].bar(range(4), pca.loadings_[:, 0], align='center') ax[1].bar(range(4), pca.loadings_[:, 1], align='center') ax[0].set_ylabel('Factor loading onto PC1') ax[1].set_ylabel('Factor loading onto PC2') ax[0].set_xticks(range(4)) ax[1].set_xticks(range(4)) ax[0].set_xticklabels(xlabels, rotation=45) ax[1].set_xticklabels(xlabels, rotation=45) plt.ylim([-1, 1]) plt.tight_layout() For instance, we may say that most of the variance in the first component is attributed to the petal features (although the loading of sepal length on PC1 is also not much less in magnitude). In contrast, the remaining variance captured by PC2 is mostly due to the sepal width. Note that we know from Example 2 that PC1 explains most of the variance, and based on the information from the loading plots, we may say that petal features combined with sepal length may explain most of the spread in the data.","title":"Example 4 - Factor Loadings"},{"location":"user_guide/feature_extraction/PrincipalComponentAnalysis/#example-5-feature-extraction-pipeline","text":"from sklearn.pipeline import make_pipeline from sklearn.neighbors import KNeighborsClassifier from sklearn.preprocessing import StandardScaler from sklearn.model_selection import train_test_split from mlxtend.data import wine_data X, y = wine_data() X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=123, test_size=0.3, stratify=y) pipe_pca = make_pipeline(StandardScaler(), PrincipalComponentAnalysis(n_components=3), KNeighborsClassifier(n_neighbors=5)) pipe_pca.fit(X_train, y_train) print('Transf. training accyracy: %.2f%%' % (pipe_pca.score(X_train, y_train)*100)) print('Transf. test accyracy: %.2f%%' % (pipe_pca.score(X_test, y_test)*100)) Transf. training accyracy: 96.77% Transf. test accyracy: 96.30%","title":"Example 5 - Feature Extraction Pipeline"},{"location":"user_guide/feature_extraction/PrincipalComponentAnalysis/#example-6-whitening","text":"Certain algorithms require the data to be whitened. This means that the features have unit variance and the off-diagonals are all zero (i.e., the features are uncorrelated). PCA already ensures that the features are uncorrelated, hence, we only need to apply a simple scaling to whiten the transformed data. For instance, for a given transformed feature X'_i , we divide it by the square-root of the corresponding eigenvalue \\lambda_i : X'_{\\text{whitened}} = \\frac{X'_i}{\\sqrt{\\lambda_i}}. The whitening via the PrincipalComponentAnalysis can be achieved by setting whitening=True during initialization. Let's demonstrate that with an example. from sklearn.preprocessing import StandardScaler from sklearn.model_selection import train_test_split from mlxtend.data import wine_data X, y = wine_data() X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=123, test_size=0.3, stratify=y)","title":"Example 6 - Whitening"},{"location":"user_guide/feature_extraction/PrincipalComponentAnalysis/#regular-pca","text":"sc = StandardScaler() pca1 = PrincipalComponentAnalysis(n_components=2) X_train_scaled = sc.fit_transform(X_train) X_train_transf = pca1.fit(X_train_scaled).transform(X_train_scaled) with plt.style.context('seaborn-whitegrid'): plt.figure(figsize=(6, 4)) for lab, col in zip((0, 1, 2), ('blue', 'red', 'green')): plt.scatter(X_train_transf[y_train==lab, 0], X_train_transf[y_train==lab, 1], label=lab, c=col) plt.xlabel('Principal Component 1') plt.ylabel('Principal Component 2') plt.legend(loc='lower center') plt.tight_layout() plt.show() np.set_printoptions(precision=1, suppress=True) print('Covariance matrix:\\n') np.cov(X_train_transf.T) Covariance matrix: array([[4.9, 0. ], [0. , 2.5]]) As we can see, the features are uncorrelated after transformation but don't have unit variance.","title":"Regular PCA"},{"location":"user_guide/feature_extraction/PrincipalComponentAnalysis/#pca-with-whitening","text":"sc = StandardScaler() pca1 = PrincipalComponentAnalysis(n_components=2, whitening=True) X_train_scaled = sc.fit_transform(X_train) X_train_transf = pca1.fit(X_train_scaled).transform(X_train_scaled) with plt.style.context('seaborn-whitegrid'): plt.figure(figsize=(6, 4)) for lab, col in zip((0, 1, 2), ('blue', 'red', 'green')): plt.scatter(X_train_transf[y_train==lab, 0], X_train_transf[y_train==lab, 1], label=lab, c=col) plt.xlabel('Principal Component 1') plt.ylabel('Principal Component 2') plt.legend(loc='lower center') plt.tight_layout() plt.show() np.set_printoptions(precision=1, suppress=True) print('Covariance matrix:\\n') np.cov(X_train_transf.T) Covariance matrix: array([[1., 0.], [0., 1.]]) As we can see above, the whitening achieves that all features now have unit variance. I.e., the covariance matrix of the transformed features becomes the identity matrix.","title":"PCA with Whitening"},{"location":"user_guide/feature_extraction/PrincipalComponentAnalysis/#api","text":"PrincipalComponentAnalysis(n_components=None, solver='svd', whitening=False) Principal Component Analysis Class Parameters n_components : int (default: None) The number of principal components for transformation. Keeps the original dimensions of the dataset if None . solver : str (default: 'svd') Method for performing the matrix decomposition. {'eigen', 'svd'} whitening : bool (default: False) Performs whitening such that the covariance matrix of the transformed data will be the identity matrix. Attributes w_ : array-like, shape=[n_features, n_components] Projection matrix e_vals_ : array-like, shape=[n_features] Eigenvalues in sorted order. e_vecs_ : array-like, shape=[n_features] Eigenvectors in sorted order. e_vals_normalized_ : array-like, shape=[n_features] Normalized eigen values such that they sum up to 1. This is equal to what's often referred to as \"explained variance ratios.\" loadings_ : array_like, shape=[n_features, n_features] The factor loadings of the original variables onto the principal components. The columns are the principal components, and the rows are the features loadings. For instance, the first column contains the loadings onto the first principal component. Note that the signs may be flipped depending on whether you use the 'eigen' or 'svd' solver; this does not affect the interpretation of the loadings though. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/feature_extraction/PrincipalComponentAnalysis/","title":"API"},{"location":"user_guide/feature_extraction/PrincipalComponentAnalysis/#methods","text":"fit(X, y=None) Learn model from training data. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns self : object get_params(deep=True) Get parameters for this estimator. Parameters deep : boolean, optional If True, will return the parameters for this estimator and contained subobjects that are estimators. Returns params : mapping of string to any Parameter names mapped to their values.' adapted from https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/base.py Author: Gael Varoquaux gael.varoquaux@normalesup.org License: BSD 3 clause set_params( params) Set the parameters of this estimator. The method works on simple estimators as well as on nested objects (such as pipelines). The latter have parameters of the form <component>__<parameter> so that it's possible to update each component of a nested object. Returns self adapted from https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/base.py Author: Gael Varoquaux gael.varoquaux@normalesup.org License: BSD 3 clause transform(X) Apply the linear transformation on X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns X_projected : np.ndarray, shape = [n_samples, n_components] Projected training vectors.","title":"Methods"},{"location":"user_guide/feature_extraction/RBFKernelPCA/","text":"RBFKernelPCA: RBF kernel principal component analysis Implementation of RBF Kernel Principal Component Analysis for non-linear dimensionality reduction from mlxtend.feature_extraction import RBFKernelPCA Overview Most machine learning algorithms have been developed and statistically validated for linearly separable data. Popular examples are linear classifiers like Support Vector Machines (SVMs) or the (standard) Principal Component Analysis (PCA) for dimensionality reduction. However, most real world data requires nonlinear methods in order to perform tasks that involve the analysis and discovery of patterns successfully. The focus of this overview is to briefly introduce the idea of kernel methods and to implement a Gaussian radius basis function (RBF) kernel that is used to perform nonlinear dimensionality reduction via BF kernel principal component analysis (kPCA). Principal Component Analysis The main purpose of principal component analysis (PCA) is the analysis of data to identify patterns that represent the data \u201cwell.\u201d The principal components can be understood as new axes of the dataset that maximize the variance along those axes (the eigenvectors of the covariance matrix). In other words, PCA aims to find the axes with maximum variances along which the data is most spread. For more details, please see the related article on mlxtend.feature_extraction.PrincipalComponentAnalysis . Nonlinear dimensionality reduction The \u201cclassic\u201d PCA approach described above is a linear projection technique that works well if the data is linearly separable. However, in the case of linearly inseparable data, a nonlinear technique is required if the task is to reduce the dimensionality of a dataset. Kernel functions and the kernel trick The basic idea to deal with linearly inseparable data is to project it onto a higher dimensional space where it becomes linearly separable. Let us call this nonlinear mapping function \\phi so that the mapping of a sample \\mathbf{x} can be written as \\mathbf{x} \\rightarrow \\phi (\\mathbf{x}) , which is called \"kernel function.\" Now, the term \"kernel\" describes a function that calculates the dot product of the images of the samples \\mathbf{x} under \\phi . \\kappa(\\mathbf{x_i, x_j}) = \\phi (\\mathbf{x_i}) \\phi (\\mathbf{x_j})^T More details about the derivation of this equation are provided in this excellent review article by Quan Wang: Kernel Principal Component Analysis and its Applications in Face Recognition and Active Shape Models .[ 1 ] In other words, the function \\phi maps the original d-dimensional features into a larger, k-dimensional feature space by creating nononlinear combinations of the original features. For example, if \\mathbf{x} consists of 2 features: \\mathbf{x} = \\big[x_1 \\quad x_2\\big]^T \\quad \\quad \\mathbf{x} \\in I\\!R^d \\Downarrow \\phi \\mathbf{x}' = \\big[x_1 \\quad x_2 \\quad x_1 x_2 \\quad x_{1}^2 \\quad x_1 x_{2}^3 \\quad \\dots \\big]^T \\quad \\quad \\mathbf{x} \\in I\\!R^k (k >> d) Often, the mathematical definition of the RBF kernel is written and implemented as \\kappa(\\mathbf{x_i, x_j}) = exp\\bigg(- \\gamma \\; \\lVert\\mathbf{x_i - x_j }\\rVert^{2}_{2} \\bigg) where \\textstyle\\gamma = \\tfrac{1}{2\\sigma^2} is a free parameter that is to be optimized. Gaussian radial basis function (RBF) Kernel PCA In the linear PCA approach, we are interested in the principal components that maximize the variance in the dataset. This is done by extracting the eigenvectors (principle components) that correspond to the largest eigenvalues based on the covariance matrix: \\text{Cov} = \\frac{1}{N} \\sum_{i=1}^{N} \\mathbf{x_i} \\mathbf{x_i}^T Bernhard Scholkopf ( Kernel Principal Component Analysis [ 2 ]) generalized this approach for data that was mapped onto the higher dimensional space via a kernel function: \\text{Cov} = \\frac{1}{N} \\sum_{i=1}^{N} \\phi(\\mathbf{x_i}) \\phi(\\mathbf{x_i})^T However, in practice the the covariance matrix in the higher dimensional space is not calculated explicitly (kernel trick). Therefore, the implementation of RBF kernel PCA does not yield the principal component axes (in contrast to the standard PCA), but the obtained eigenvectors can be understood as projections of the data onto the principal components. RBF kernel PCA step-by-step 1. Computation of the kernel (similarity) matrix. In this first step, we need to calculate \\kappa(\\mathbf{x_i, x_j}) = exp\\bigg(- \\gamma \\; \\lVert\\mathbf{x_i - x_j }\\rVert^{2}_{2} \\bigg) for every pair of points. E.g., if we have a dataset of 100 samples, this step would result in a symmetric 100x100 kernel matrix. 2. Eigendecomposition of the kernel matrix. Since it is not guaranteed that the kernel matrix is centered, we can apply the following equation to do so: K' = K - \\mathbf{1_N} K - K \\mathbf{1_N} + \\mathbf{1_N} K \\mathbf{1_N} where \\mathbf{1_N} is (like the kernel matrix) a N\\times N matrix with all values equal to \\frac{1}{N} . [ 3 ] Now, we have to obtain the eigenvectors of the centered kernel matrix that correspond to the largest eigenvalues. Those eigenvectors are the data points already projected onto the respective principal components. Projecting new data So far, so good, in the sections above, we have been projecting an dataset onto a new feature subspace. However, in a real application, we are usually interested in mapping new data points onto the same new feature subspace (e.g., if are working with a training and a test dataset in pattern classification tasks). Remember, when we computed the eigenvectors \\mathbf{\\alpha} of the centered kernel matrix, those values were actually already the projected datapoints onto the principal component axis \\mathbf{g} . If we want to project a new data point \\mathbf{x} onto this principal component axis, we'd need to compute \\phi(\\mathbf{x})^T \\mathbf{g} . Fortunately, also here, we don't have to compute \\phi(\\mathbf{x})^T \\mathbf{g} explicitely but use the kernel trick to calculate the RBF kernel between the new data point and every data point j in the training dataset: \\phi(\\mathbf{x})^T \\mathbf{g} = \\sum_j \\alpha_{i} \\; \\phi(\\mathbf{x}) \\; \\phi(\\mathbf{x_j})^T = \\sum_j \\alpha_{i} \\; \\kappa(\\mathbf{x}, \\mathbf{x_j}) and the eigenvectors \\alpha and eigenvalues \\lambda of the Kernel matrix \\mathbf{K} satisfy the equation \\mathbf{K} \\alpha = \\lambda \\alpha , we just need to normalize the eigenvector by the corresponding eigenvalue. References [1] Q. Wang. Kernel principal component analysis and its applications in face recognition and active shape models . CoRR, abs/1207.3538, 2012. [2] B. Scholkopf, A. Smola, and K.-R. Muller. Kernel principal component analysis . pages 583\u2013588, 1997. [3] B. Scholkopf, A. Smola, and K.-R. Muller. Nonlinear component analysis as a kernel eigenvalue problem . Neural computation, 10(5):1299\u20131319, 1998. Example 1 - Half-moon shapes We will start with a simple example of 2 half-moon shapes generated by the make_moons function from scikit-learn. import matplotlib.pyplot as plt from sklearn.datasets import make_moons X, y = make_moons(n_samples=50, random_state=1) plt.scatter(X[y==0, 0], X[y==0, 1], color='red', marker='o', alpha=0.5) plt.scatter(X[y==1, 0], X[y==1, 1], color='blue', marker='^', alpha=0.5) plt.ylabel('y coordinate') plt.xlabel('x coordinate') plt.show() Since the two half-moon shapes are linearly inseparable, we expect that the \u201cclassic\u201d PCA will fail to give us a \u201cgood\u201d representation of the data in 1D space. Let us use PCA class to perform the dimensionality reduction. from mlxtend.feature_extraction import PrincipalComponentAnalysis as PCA pca = PCA(n_components=2) X_pca = pca.fit(X).transform(X) plt.scatter(X_pca[y==0, 0], X_pca[y==0, 1], color='red', marker='o', alpha=0.5) plt.scatter(X_pca[y==1, 0], X_pca[y==1, 1], color='blue', marker='^', alpha=0.5) plt.xlabel('PC1') plt.ylabel('PC2') plt.show() As we can see, the resulting principal components do not yield a subspace where the data is linearly separated well. Note that PCA is a unsupervised method and does not \u201cconsider\u201d class labels in order to maximize the variance in contrast to Linear Discriminant Analysis. Here, the colors blue and red are just added for visualization purposes to indicate the degree of separation. Next, we will perform dimensionality reduction via RBF kernel PCA on our half-moon data. The choice of \\gamma depends on the dataset and can be obtained via hyperparameter tuning techniques like Grid Search. Hyperparameter tuning is a broad topic itself, and here I will just use a \\gamma -value that I found to produce \u201cgood\u201d results. from mlxtend.data import iris_data from mlxtend.preprocessing import standardize from mlxtend.feature_extraction import RBFKernelPCA as KPCA kpca = KPCA(gamma=15.0, n_components=2) kpca.fit(X) X_kpca = kpca.X_projected_ Please note that the components of kernel methods such as RBF kernel PCA already represent the projected data points (in contrast to PCA, where the component axis are the \"top k\" eigenvectors thar are used to contruct a projection matrix, which is then used to transform the training samples). Thus, the projected training set is available after fitting via the .X_projected_ attribute. plt.scatter(X_kpca[y==0, 0], X_kpca[y==0, 1], color='red', marker='o', alpha=0.5) plt.scatter(X_kpca[y==1, 0], X_kpca[y==1, 1], color='blue', marker='^', alpha=0.5) plt.title('First 2 principal components after RBF Kernel PCA') plt.xlabel('PC1') plt.ylabel('PC2') plt.show() The new feature space is linearly separable now. Since we are often interested in dimensionality reduction, let's have a look at the first component only. import numpy as np plt.scatter(X_kpca[y==0, 0], np.zeros((25, 1)), color='red', marker='o', alpha=0.5) plt.scatter(X_kpca[y==1, 0], np.zeros((25, 1)), color='blue', marker='^', alpha=0.5) plt.title('First principal component after RBF Kernel PCA') plt.xlabel('PC1') plt.yticks([]) plt.show() We can clearly see that the projection via RBF kernel PCA yielded a subspace where the classes are separated well. Such a subspace can then be used as input for generalized linear classification models, e.g., logistic regression. Projecting new data Finally, via the transform method, we can project new data onto the new component axes. import matplotlib.pyplot as plt from sklearn.datasets import make_moons X2, y2 = make_moons(n_samples=200, random_state=5) X2_kpca = kpca.transform(X2) plt.scatter(X_kpca[y==0, 0], X_kpca[y==0, 1], color='red', marker='o', alpha=0.5, label='fit data') plt.scatter(X_kpca[y==1, 0], X_kpca[y==1, 1], color='blue', marker='^', alpha=0.5, label='fit data') plt.scatter(X2_kpca[y2==0, 0], X2_kpca[y2==0, 1], color='orange', marker='v', alpha=0.2, label='new data') plt.scatter(X2_kpca[y2==1, 0], X2_kpca[y2==1, 1], color='cyan', marker='s', alpha=0.2, label='new data') plt.legend() plt.show() Example 2 - Concentric circles Following the concepts explained in example 1, let's have a look at another classic case: 2 concentric circles with random noise produced by scikit-learn\u2019s make_circles . from sklearn.datasets import make_circles X, y = make_circles(n_samples=1000, random_state=123, noise=0.1, factor=0.2) plt.figure(figsize=(8,6)) plt.scatter(X[y==0, 0], X[y==0, 1], color='red', alpha=0.5) plt.scatter(X[y==1, 0], X[y==1, 1], color='blue', alpha=0.5) plt.title('Concentric circles') plt.ylabel('y coordinate') plt.xlabel('x coordinate') plt.show() from mlxtend.data import iris_data from mlxtend.preprocessing import standardize from mlxtend.feature_extraction import RBFKernelPCA as KPCA kpca = KPCA(gamma=15.0, n_components=2) kpca.fit(X) X_kpca = kpca.X_projected_ plt.scatter(X_kpca[y==0, 0], X_kpca[y==0, 1], color='red', marker='o', alpha=0.5) plt.scatter(X_kpca[y==1, 0], X_kpca[y==1, 1], color='blue', marker='^', alpha=0.5) plt.title('First 2 principal components after RBF Kernel PCA') plt.xlabel('PC1') plt.ylabel('PC2') plt.show() plt.scatter(X_kpca[y==0, 0], np.zeros((500, 1)), color='red', marker='o', alpha=0.5) plt.scatter(X_kpca[y==1, 0], np.zeros((500, 1)), color='blue', marker='^', alpha=0.5) plt.title('First principal component after RBF Kernel PCA') plt.xlabel('PC1') plt.yticks([]) plt.show() API RBFKernelPCA(gamma=15.0, n_components=None, copy_X=True) RBF Kernel Principal Component Analysis for dimensionality reduction. Parameters gamma : float (default: 15.0) Free parameter (coefficient) of the RBF kernel. n_components : int (default: None) The number of principal components for transformation. Keeps the original dimensions of the dataset if None . copy_X : bool (default: True) Copies training data, which is required to compute the projection of new data via the transform method. Uses a reference to X if False. Attributes e_vals_ : array-like, shape=[n_features] Eigenvalues in sorted order. e_vecs_ : array-like, shape=[n_features] Eigenvectors in sorted order. X_projected_ : array-like, shape=[n_samples, n_components] Training samples projected along the component axes. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/feature_extraction/RBFKernelPCA/ Methods fit(X) Learn model from training data. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns self : object transform(X) Apply the non-linear transformation on X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns X_projected : np.ndarray, shape = [n_samples, n_components] Projected training vectors.","title":"RBFKernelPCA"},{"location":"user_guide/feature_extraction/RBFKernelPCA/#rbfkernelpca-rbf-kernel-principal-component-analysis","text":"Implementation of RBF Kernel Principal Component Analysis for non-linear dimensionality reduction from mlxtend.feature_extraction import RBFKernelPCA","title":"RBFKernelPCA: RBF kernel principal component analysis"},{"location":"user_guide/feature_extraction/RBFKernelPCA/#overview","text":"Most machine learning algorithms have been developed and statistically validated for linearly separable data. Popular examples are linear classifiers like Support Vector Machines (SVMs) or the (standard) Principal Component Analysis (PCA) for dimensionality reduction. However, most real world data requires nonlinear methods in order to perform tasks that involve the analysis and discovery of patterns successfully. The focus of this overview is to briefly introduce the idea of kernel methods and to implement a Gaussian radius basis function (RBF) kernel that is used to perform nonlinear dimensionality reduction via BF kernel principal component analysis (kPCA).","title":"Overview"},{"location":"user_guide/feature_extraction/RBFKernelPCA/#principal-component-analysis","text":"The main purpose of principal component analysis (PCA) is the analysis of data to identify patterns that represent the data \u201cwell.\u201d The principal components can be understood as new axes of the dataset that maximize the variance along those axes (the eigenvectors of the covariance matrix). In other words, PCA aims to find the axes with maximum variances along which the data is most spread. For more details, please see the related article on mlxtend.feature_extraction.PrincipalComponentAnalysis .","title":"Principal Component Analysis"},{"location":"user_guide/feature_extraction/RBFKernelPCA/#nonlinear-dimensionality-reduction","text":"The \u201cclassic\u201d PCA approach described above is a linear projection technique that works well if the data is linearly separable. However, in the case of linearly inseparable data, a nonlinear technique is required if the task is to reduce the dimensionality of a dataset.","title":"Nonlinear dimensionality reduction"},{"location":"user_guide/feature_extraction/RBFKernelPCA/#kernel-functions-and-the-kernel-trick","text":"The basic idea to deal with linearly inseparable data is to project it onto a higher dimensional space where it becomes linearly separable. Let us call this nonlinear mapping function \\phi so that the mapping of a sample \\mathbf{x} can be written as \\mathbf{x} \\rightarrow \\phi (\\mathbf{x}) , which is called \"kernel function.\" Now, the term \"kernel\" describes a function that calculates the dot product of the images of the samples \\mathbf{x} under \\phi . \\kappa(\\mathbf{x_i, x_j}) = \\phi (\\mathbf{x_i}) \\phi (\\mathbf{x_j})^T More details about the derivation of this equation are provided in this excellent review article by Quan Wang: Kernel Principal Component Analysis and its Applications in Face Recognition and Active Shape Models .[ 1 ] In other words, the function \\phi maps the original d-dimensional features into a larger, k-dimensional feature space by creating nononlinear combinations of the original features. For example, if \\mathbf{x} consists of 2 features: \\mathbf{x} = \\big[x_1 \\quad x_2\\big]^T \\quad \\quad \\mathbf{x} \\in I\\!R^d \\Downarrow \\phi \\mathbf{x}' = \\big[x_1 \\quad x_2 \\quad x_1 x_2 \\quad x_{1}^2 \\quad x_1 x_{2}^3 \\quad \\dots \\big]^T \\quad \\quad \\mathbf{x} \\in I\\!R^k (k >> d) Often, the mathematical definition of the RBF kernel is written and implemented as \\kappa(\\mathbf{x_i, x_j}) = exp\\bigg(- \\gamma \\; \\lVert\\mathbf{x_i - x_j }\\rVert^{2}_{2} \\bigg) where \\textstyle\\gamma = \\tfrac{1}{2\\sigma^2} is a free parameter that is to be optimized.","title":"Kernel functions and the kernel trick"},{"location":"user_guide/feature_extraction/RBFKernelPCA/#gaussian-radial-basis-function-rbf-kernel-pca","text":"In the linear PCA approach, we are interested in the principal components that maximize the variance in the dataset. This is done by extracting the eigenvectors (principle components) that correspond to the largest eigenvalues based on the covariance matrix: \\text{Cov} = \\frac{1}{N} \\sum_{i=1}^{N} \\mathbf{x_i} \\mathbf{x_i}^T Bernhard Scholkopf ( Kernel Principal Component Analysis [ 2 ]) generalized this approach for data that was mapped onto the higher dimensional space via a kernel function: \\text{Cov} = \\frac{1}{N} \\sum_{i=1}^{N} \\phi(\\mathbf{x_i}) \\phi(\\mathbf{x_i})^T However, in practice the the covariance matrix in the higher dimensional space is not calculated explicitly (kernel trick). Therefore, the implementation of RBF kernel PCA does not yield the principal component axes (in contrast to the standard PCA), but the obtained eigenvectors can be understood as projections of the data onto the principal components.","title":"Gaussian radial basis function (RBF) Kernel PCA"},{"location":"user_guide/feature_extraction/RBFKernelPCA/#rbf-kernel-pca-step-by-step","text":"","title":"RBF kernel PCA step-by-step"},{"location":"user_guide/feature_extraction/RBFKernelPCA/#1-computation-of-the-kernel-similarity-matrix","text":"In this first step, we need to calculate \\kappa(\\mathbf{x_i, x_j}) = exp\\bigg(- \\gamma \\; \\lVert\\mathbf{x_i - x_j }\\rVert^{2}_{2} \\bigg) for every pair of points. E.g., if we have a dataset of 100 samples, this step would result in a symmetric 100x100 kernel matrix.","title":"1. Computation of the kernel (similarity) matrix."},{"location":"user_guide/feature_extraction/RBFKernelPCA/#2-eigendecomposition-of-the-kernel-matrix","text":"Since it is not guaranteed that the kernel matrix is centered, we can apply the following equation to do so: K' = K - \\mathbf{1_N} K - K \\mathbf{1_N} + \\mathbf{1_N} K \\mathbf{1_N} where \\mathbf{1_N} is (like the kernel matrix) a N\\times N matrix with all values equal to \\frac{1}{N} . [ 3 ] Now, we have to obtain the eigenvectors of the centered kernel matrix that correspond to the largest eigenvalues. Those eigenvectors are the data points already projected onto the respective principal components.","title":"2. Eigendecomposition of the kernel matrix."},{"location":"user_guide/feature_extraction/RBFKernelPCA/#projecting-new-data","text":"So far, so good, in the sections above, we have been projecting an dataset onto a new feature subspace. However, in a real application, we are usually interested in mapping new data points onto the same new feature subspace (e.g., if are working with a training and a test dataset in pattern classification tasks). Remember, when we computed the eigenvectors \\mathbf{\\alpha} of the centered kernel matrix, those values were actually already the projected datapoints onto the principal component axis \\mathbf{g} . If we want to project a new data point \\mathbf{x} onto this principal component axis, we'd need to compute \\phi(\\mathbf{x})^T \\mathbf{g} . Fortunately, also here, we don't have to compute \\phi(\\mathbf{x})^T \\mathbf{g} explicitely but use the kernel trick to calculate the RBF kernel between the new data point and every data point j in the training dataset: \\phi(\\mathbf{x})^T \\mathbf{g} = \\sum_j \\alpha_{i} \\; \\phi(\\mathbf{x}) \\; \\phi(\\mathbf{x_j})^T = \\sum_j \\alpha_{i} \\; \\kappa(\\mathbf{x}, \\mathbf{x_j}) and the eigenvectors \\alpha and eigenvalues \\lambda of the Kernel matrix \\mathbf{K} satisfy the equation \\mathbf{K} \\alpha = \\lambda \\alpha , we just need to normalize the eigenvector by the corresponding eigenvalue.","title":"Projecting new data"},{"location":"user_guide/feature_extraction/RBFKernelPCA/#references","text":"[1] Q. Wang. Kernel principal component analysis and its applications in face recognition and active shape models . CoRR, abs/1207.3538, 2012. [2] B. Scholkopf, A. Smola, and K.-R. Muller. Kernel principal component analysis . pages 583\u2013588, 1997. [3] B. Scholkopf, A. Smola, and K.-R. Muller. Nonlinear component analysis as a kernel eigenvalue problem . Neural computation, 10(5):1299\u20131319, 1998.","title":"References"},{"location":"user_guide/feature_extraction/RBFKernelPCA/#example-1-half-moon-shapes","text":"We will start with a simple example of 2 half-moon shapes generated by the make_moons function from scikit-learn. import matplotlib.pyplot as plt from sklearn.datasets import make_moons X, y = make_moons(n_samples=50, random_state=1) plt.scatter(X[y==0, 0], X[y==0, 1], color='red', marker='o', alpha=0.5) plt.scatter(X[y==1, 0], X[y==1, 1], color='blue', marker='^', alpha=0.5) plt.ylabel('y coordinate') plt.xlabel('x coordinate') plt.show() Since the two half-moon shapes are linearly inseparable, we expect that the \u201cclassic\u201d PCA will fail to give us a \u201cgood\u201d representation of the data in 1D space. Let us use PCA class to perform the dimensionality reduction. from mlxtend.feature_extraction import PrincipalComponentAnalysis as PCA pca = PCA(n_components=2) X_pca = pca.fit(X).transform(X) plt.scatter(X_pca[y==0, 0], X_pca[y==0, 1], color='red', marker='o', alpha=0.5) plt.scatter(X_pca[y==1, 0], X_pca[y==1, 1], color='blue', marker='^', alpha=0.5) plt.xlabel('PC1') plt.ylabel('PC2') plt.show() As we can see, the resulting principal components do not yield a subspace where the data is linearly separated well. Note that PCA is a unsupervised method and does not \u201cconsider\u201d class labels in order to maximize the variance in contrast to Linear Discriminant Analysis. Here, the colors blue and red are just added for visualization purposes to indicate the degree of separation. Next, we will perform dimensionality reduction via RBF kernel PCA on our half-moon data. The choice of \\gamma depends on the dataset and can be obtained via hyperparameter tuning techniques like Grid Search. Hyperparameter tuning is a broad topic itself, and here I will just use a \\gamma -value that I found to produce \u201cgood\u201d results. from mlxtend.data import iris_data from mlxtend.preprocessing import standardize from mlxtend.feature_extraction import RBFKernelPCA as KPCA kpca = KPCA(gamma=15.0, n_components=2) kpca.fit(X) X_kpca = kpca.X_projected_ Please note that the components of kernel methods such as RBF kernel PCA already represent the projected data points (in contrast to PCA, where the component axis are the \"top k\" eigenvectors thar are used to contruct a projection matrix, which is then used to transform the training samples). Thus, the projected training set is available after fitting via the .X_projected_ attribute. plt.scatter(X_kpca[y==0, 0], X_kpca[y==0, 1], color='red', marker='o', alpha=0.5) plt.scatter(X_kpca[y==1, 0], X_kpca[y==1, 1], color='blue', marker='^', alpha=0.5) plt.title('First 2 principal components after RBF Kernel PCA') plt.xlabel('PC1') plt.ylabel('PC2') plt.show() The new feature space is linearly separable now. Since we are often interested in dimensionality reduction, let's have a look at the first component only. import numpy as np plt.scatter(X_kpca[y==0, 0], np.zeros((25, 1)), color='red', marker='o', alpha=0.5) plt.scatter(X_kpca[y==1, 0], np.zeros((25, 1)), color='blue', marker='^', alpha=0.5) plt.title('First principal component after RBF Kernel PCA') plt.xlabel('PC1') plt.yticks([]) plt.show() We can clearly see that the projection via RBF kernel PCA yielded a subspace where the classes are separated well. Such a subspace can then be used as input for generalized linear classification models, e.g., logistic regression.","title":"Example 1 - Half-moon shapes"},{"location":"user_guide/feature_extraction/RBFKernelPCA/#projecting-new-data_1","text":"Finally, via the transform method, we can project new data onto the new component axes. import matplotlib.pyplot as plt from sklearn.datasets import make_moons X2, y2 = make_moons(n_samples=200, random_state=5) X2_kpca = kpca.transform(X2) plt.scatter(X_kpca[y==0, 0], X_kpca[y==0, 1], color='red', marker='o', alpha=0.5, label='fit data') plt.scatter(X_kpca[y==1, 0], X_kpca[y==1, 1], color='blue', marker='^', alpha=0.5, label='fit data') plt.scatter(X2_kpca[y2==0, 0], X2_kpca[y2==0, 1], color='orange', marker='v', alpha=0.2, label='new data') plt.scatter(X2_kpca[y2==1, 0], X2_kpca[y2==1, 1], color='cyan', marker='s', alpha=0.2, label='new data') plt.legend() plt.show()","title":"Projecting new data"},{"location":"user_guide/feature_extraction/RBFKernelPCA/#example-2-concentric-circles","text":"Following the concepts explained in example 1, let's have a look at another classic case: 2 concentric circles with random noise produced by scikit-learn\u2019s make_circles . from sklearn.datasets import make_circles X, y = make_circles(n_samples=1000, random_state=123, noise=0.1, factor=0.2) plt.figure(figsize=(8,6)) plt.scatter(X[y==0, 0], X[y==0, 1], color='red', alpha=0.5) plt.scatter(X[y==1, 0], X[y==1, 1], color='blue', alpha=0.5) plt.title('Concentric circles') plt.ylabel('y coordinate') plt.xlabel('x coordinate') plt.show() from mlxtend.data import iris_data from mlxtend.preprocessing import standardize from mlxtend.feature_extraction import RBFKernelPCA as KPCA kpca = KPCA(gamma=15.0, n_components=2) kpca.fit(X) X_kpca = kpca.X_projected_ plt.scatter(X_kpca[y==0, 0], X_kpca[y==0, 1], color='red', marker='o', alpha=0.5) plt.scatter(X_kpca[y==1, 0], X_kpca[y==1, 1], color='blue', marker='^', alpha=0.5) plt.title('First 2 principal components after RBF Kernel PCA') plt.xlabel('PC1') plt.ylabel('PC2') plt.show() plt.scatter(X_kpca[y==0, 0], np.zeros((500, 1)), color='red', marker='o', alpha=0.5) plt.scatter(X_kpca[y==1, 0], np.zeros((500, 1)), color='blue', marker='^', alpha=0.5) plt.title('First principal component after RBF Kernel PCA') plt.xlabel('PC1') plt.yticks([]) plt.show()","title":"Example 2 - Concentric circles"},{"location":"user_guide/feature_extraction/RBFKernelPCA/#api","text":"RBFKernelPCA(gamma=15.0, n_components=None, copy_X=True) RBF Kernel Principal Component Analysis for dimensionality reduction. Parameters gamma : float (default: 15.0) Free parameter (coefficient) of the RBF kernel. n_components : int (default: None) The number of principal components for transformation. Keeps the original dimensions of the dataset if None . copy_X : bool (default: True) Copies training data, which is required to compute the projection of new data via the transform method. Uses a reference to X if False. Attributes e_vals_ : array-like, shape=[n_features] Eigenvalues in sorted order. e_vecs_ : array-like, shape=[n_features] Eigenvectors in sorted order. X_projected_ : array-like, shape=[n_samples, n_components] Training samples projected along the component axes. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/feature_extraction/RBFKernelPCA/","title":"API"},{"location":"user_guide/feature_extraction/RBFKernelPCA/#methods","text":"fit(X) Learn model from training data. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns self : object transform(X) Apply the non-linear transformation on X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns X_projected : np.ndarray, shape = [n_samples, n_components] Projected training vectors.","title":"Methods"},{"location":"user_guide/feature_selection/ColumnSelector/","text":"ColumnSelector: Scikit-learn utility function to select specific columns in a pipeline Implementation of a column selector class for scikit-learn pipelines. from mlxtend.feature_selection import ColumnSelector Overview The ColumnSelector can be used for \"manual\" feature selection, e.g., as part of a grid search via a scikit-learn pipeline. References - Example 1 - Fitting an Estimator on a Feature Subset Load a simple benchmark dataset: from sklearn.datasets import load_iris iris = load_iris() X = iris.data y = iris.target The ColumnSelector is a simple transformer class that selects specific columns (features) from a datast. For instance, using the transform method returns a reduced dataset that only contains two features (here: the first two features via the indices 0 and 1, respectively): from mlxtend.feature_selection import ColumnSelector col_selector = ColumnSelector(cols=(0, 1)) # col_selector.fit(X) # optional, does not do anything col_selector.transform(X).shape (150, 2) ColumnSelector works both with numpy arrays and pandas dataframes: import pandas as pd iris_df = pd.DataFrame(iris.data, columns=iris.feature_names) iris_df.head() .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } sepal length (cm) sepal width (cm) petal length (cm) petal width (cm) 0 5.1 3.5 1.4 0.2 1 4.9 3.0 1.4 0.2 2 4.7 3.2 1.3 0.2 3 4.6 3.1 1.5 0.2 4 5.0 3.6 1.4 0.2 col_selector = ColumnSelector(cols=(\"sepal length (cm)\", \"sepal width (cm)\")) col_selector.transform(iris_df).shape (150, 2) Similarly, we can use the ColumnSelector as part of a scikit-learn Pipeline : from sklearn.neighbors import KNeighborsClassifier from sklearn.preprocessing import StandardScaler from sklearn.pipeline import make_pipeline pipe = make_pipeline(StandardScaler(), ColumnSelector(cols=(0, 1)), KNeighborsClassifier()) pipe.fit(X, y) pipe.score(X, y) 0.84 Example 2 - Feature Selection via GridSearch Example 1 showed a simple useage example of the ColumnSelector ; however, selecting columns from a dataset is trivial and does not require a specific transformer class since we could have achieved the same results via classifier.fit(X[:, :2], y) classifier.score(X[:, :2], y) However, the ColumnSelector becomes really useful for feature selection as part of a grid search as shown in this example. Load a simple benchmark dataset: from sklearn.datasets import load_iris iris = load_iris() X = iris.data y = iris.target Create all possible combinations: from itertools import combinations all_comb = [] for size in range(1, 5): all_comb += list(combinations(range(X.shape[1]), r=size)) print(all_comb) [(0,), (1,), (2,), (3,), (0, 1), (0, 2), (0, 3), (1, 2), (1, 3), (2, 3), (0, 1, 2), (0, 1, 3), (0, 2, 3), (1, 2, 3), (0, 1, 2, 3)] Feature and model selection via grid search: from mlxtend.feature_selection import ColumnSelector from sklearn.neighbors import KNeighborsClassifier from sklearn.preprocessing import StandardScaler from sklearn.model_selection import GridSearchCV from sklearn.pipeline import make_pipeline pipe = make_pipeline(StandardScaler(), ColumnSelector(), KNeighborsClassifier()) param_grid = {'columnselector__cols': all_comb, 'kneighborsclassifier__n_neighbors': list(range(1, 11))} grid = GridSearchCV(pipe, param_grid, cv=5, n_jobs=-1) grid.fit(X, y) print('Best parameters:', grid.best_params_) print('Best performance:', grid.best_score_) Best parameters: {'columnselector__cols': (2, 3), 'kneighborsclassifier__n_neighbors': 1} Best performance: 0.98 Example 3 -- Scaling of a Subset of Features in a scikit-learn Pipeline The following example illustrates how we could use the ColumnSelector in tandem with scikit-learn's FeatureUnion to only scale certain features (in this toy example: the first and second feature only) in a datasets in a Pipeline . from mlxtend.feature_selection import ColumnSelector from sklearn.pipeline import make_pipeline from sklearn.pipeline import Pipeline from sklearn.pipeline import FeatureUnion from sklearn.preprocessing import MinMaxScaler from sklearn.neighbors import KNeighborsClassifier from mlxtend.data import iris_data X, y = iris_data() scale_pipe = make_pipeline(ColumnSelector(cols=(0, 1)), MinMaxScaler()) pipeline = Pipeline([ ('feats', FeatureUnion([ ('col_1-2', scale_pipe), ('col_3-4', ColumnSelector(cols=(2, 3))) ])), ('clf', KNeighborsClassifier()) ]) pipeline.fit(X, y) Pipeline(memory=None, steps=[('feats', FeatureUnion(n_jobs=None, transformer_list=[('col_1-2', Pipeline(memory=None, steps=[('columnselector', ColumnSelector(cols=(0, 1), drop_axis=False)), ('minmaxscaler', MinMaxScaler(copy=True, feature_range=(0, 1)))])), ('col_3-4', ColumnSelector(cols=(2, 3), drop_axis=Fa...ki', metric_params=None, n_jobs=None, n_neighbors=5, p=2, weights='uniform'))]) API ColumnSelector(cols=None, drop_axis=False) Object for selecting specific columns from a data set. Parameters cols : array-like (default: None) A list specifying the feature indices to be selected. For example, [1, 4, 5] to select the 2nd, 5th, and 6th feature columns. If None, returns all columns in the array. drop_axis : bool (default=False) Drops last axis if True and the only one column is selected. This is useful, e.g., when the ColumnSelector is used for selecting only one column and the resulting array should be fed to e.g., a scikit-learn column selector. E.g., instead of returning an array with shape (n_samples, 1), drop_axis=True will return an aray with shape (n_samples,). Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/feature_selection/ColumnSelector/ Methods fit(X, y=None) Mock method. Does nothing. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] (default: None) Returns self fit_transform(X, y=None) Return a slice of the input array. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] (default: None) Returns X_slice : shape = [n_samples, k_features] Subset of the feature space where k_features <= n_features get_params(deep=True) Get parameters for this estimator. Parameters deep : boolean, optional If True, will return the parameters for this estimator and contained subobjects that are estimators. Returns params : mapping of string to any Parameter names mapped to their values. set_params( params) Set the parameters of this estimator. The method works on simple estimators as well as on nested objects (such as pipelines). The latter have parameters of the form <component>__<parameter> so that it's possible to update each component of a nested object. Returns self transform(X, y=None) Return a slice of the input array. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] (default: None) Returns X_slice : shape = [n_samples, k_features] Subset of the feature space where k_features <= n_features","title":"ColumnSelector: Scikit-learn utility function to select specific columns in a pipeline"},{"location":"user_guide/feature_selection/ColumnSelector/#columnselector-scikit-learn-utility-function-to-select-specific-columns-in-a-pipeline","text":"Implementation of a column selector class for scikit-learn pipelines. from mlxtend.feature_selection import ColumnSelector","title":"ColumnSelector: Scikit-learn utility function to select specific columns in a pipeline"},{"location":"user_guide/feature_selection/ColumnSelector/#overview","text":"The ColumnSelector can be used for \"manual\" feature selection, e.g., as part of a grid search via a scikit-learn pipeline.","title":"Overview"},{"location":"user_guide/feature_selection/ColumnSelector/#references","text":"-","title":"References"},{"location":"user_guide/feature_selection/ColumnSelector/#example-1-fitting-an-estimator-on-a-feature-subset","text":"Load a simple benchmark dataset: from sklearn.datasets import load_iris iris = load_iris() X = iris.data y = iris.target The ColumnSelector is a simple transformer class that selects specific columns (features) from a datast. For instance, using the transform method returns a reduced dataset that only contains two features (here: the first two features via the indices 0 and 1, respectively): from mlxtend.feature_selection import ColumnSelector col_selector = ColumnSelector(cols=(0, 1)) # col_selector.fit(X) # optional, does not do anything col_selector.transform(X).shape (150, 2) ColumnSelector works both with numpy arrays and pandas dataframes: import pandas as pd iris_df = pd.DataFrame(iris.data, columns=iris.feature_names) iris_df.head() .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } sepal length (cm) sepal width (cm) petal length (cm) petal width (cm) 0 5.1 3.5 1.4 0.2 1 4.9 3.0 1.4 0.2 2 4.7 3.2 1.3 0.2 3 4.6 3.1 1.5 0.2 4 5.0 3.6 1.4 0.2 col_selector = ColumnSelector(cols=(\"sepal length (cm)\", \"sepal width (cm)\")) col_selector.transform(iris_df).shape (150, 2) Similarly, we can use the ColumnSelector as part of a scikit-learn Pipeline : from sklearn.neighbors import KNeighborsClassifier from sklearn.preprocessing import StandardScaler from sklearn.pipeline import make_pipeline pipe = make_pipeline(StandardScaler(), ColumnSelector(cols=(0, 1)), KNeighborsClassifier()) pipe.fit(X, y) pipe.score(X, y) 0.84","title":"Example 1 - Fitting an Estimator on a Feature Subset"},{"location":"user_guide/feature_selection/ColumnSelector/#example-2-feature-selection-via-gridsearch","text":"Example 1 showed a simple useage example of the ColumnSelector ; however, selecting columns from a dataset is trivial and does not require a specific transformer class since we could have achieved the same results via classifier.fit(X[:, :2], y) classifier.score(X[:, :2], y) However, the ColumnSelector becomes really useful for feature selection as part of a grid search as shown in this example. Load a simple benchmark dataset: from sklearn.datasets import load_iris iris = load_iris() X = iris.data y = iris.target Create all possible combinations: from itertools import combinations all_comb = [] for size in range(1, 5): all_comb += list(combinations(range(X.shape[1]), r=size)) print(all_comb) [(0,), (1,), (2,), (3,), (0, 1), (0, 2), (0, 3), (1, 2), (1, 3), (2, 3), (0, 1, 2), (0, 1, 3), (0, 2, 3), (1, 2, 3), (0, 1, 2, 3)] Feature and model selection via grid search: from mlxtend.feature_selection import ColumnSelector from sklearn.neighbors import KNeighborsClassifier from sklearn.preprocessing import StandardScaler from sklearn.model_selection import GridSearchCV from sklearn.pipeline import make_pipeline pipe = make_pipeline(StandardScaler(), ColumnSelector(), KNeighborsClassifier()) param_grid = {'columnselector__cols': all_comb, 'kneighborsclassifier__n_neighbors': list(range(1, 11))} grid = GridSearchCV(pipe, param_grid, cv=5, n_jobs=-1) grid.fit(X, y) print('Best parameters:', grid.best_params_) print('Best performance:', grid.best_score_) Best parameters: {'columnselector__cols': (2, 3), 'kneighborsclassifier__n_neighbors': 1} Best performance: 0.98","title":"Example 2 - Feature Selection via GridSearch"},{"location":"user_guide/feature_selection/ColumnSelector/#example-3-scaling-of-a-subset-of-features-in-a-scikit-learn-pipeline","text":"The following example illustrates how we could use the ColumnSelector in tandem with scikit-learn's FeatureUnion to only scale certain features (in this toy example: the first and second feature only) in a datasets in a Pipeline . from mlxtend.feature_selection import ColumnSelector from sklearn.pipeline import make_pipeline from sklearn.pipeline import Pipeline from sklearn.pipeline import FeatureUnion from sklearn.preprocessing import MinMaxScaler from sklearn.neighbors import KNeighborsClassifier from mlxtend.data import iris_data X, y = iris_data() scale_pipe = make_pipeline(ColumnSelector(cols=(0, 1)), MinMaxScaler()) pipeline = Pipeline([ ('feats', FeatureUnion([ ('col_1-2', scale_pipe), ('col_3-4', ColumnSelector(cols=(2, 3))) ])), ('clf', KNeighborsClassifier()) ]) pipeline.fit(X, y) Pipeline(memory=None, steps=[('feats', FeatureUnion(n_jobs=None, transformer_list=[('col_1-2', Pipeline(memory=None, steps=[('columnselector', ColumnSelector(cols=(0, 1), drop_axis=False)), ('minmaxscaler', MinMaxScaler(copy=True, feature_range=(0, 1)))])), ('col_3-4', ColumnSelector(cols=(2, 3), drop_axis=Fa...ki', metric_params=None, n_jobs=None, n_neighbors=5, p=2, weights='uniform'))])","title":"Example 3 -- Scaling of a Subset of Features in a scikit-learn Pipeline"},{"location":"user_guide/feature_selection/ColumnSelector/#api","text":"ColumnSelector(cols=None, drop_axis=False) Object for selecting specific columns from a data set. Parameters cols : array-like (default: None) A list specifying the feature indices to be selected. For example, [1, 4, 5] to select the 2nd, 5th, and 6th feature columns. If None, returns all columns in the array. drop_axis : bool (default=False) Drops last axis if True and the only one column is selected. This is useful, e.g., when the ColumnSelector is used for selecting only one column and the resulting array should be fed to e.g., a scikit-learn column selector. E.g., instead of returning an array with shape (n_samples, 1), drop_axis=True will return an aray with shape (n_samples,). Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/feature_selection/ColumnSelector/","title":"API"},{"location":"user_guide/feature_selection/ColumnSelector/#methods","text":"fit(X, y=None) Mock method. Does nothing. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] (default: None) Returns self fit_transform(X, y=None) Return a slice of the input array. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] (default: None) Returns X_slice : shape = [n_samples, k_features] Subset of the feature space where k_features <= n_features get_params(deep=True) Get parameters for this estimator. Parameters deep : boolean, optional If True, will return the parameters for this estimator and contained subobjects that are estimators. Returns params : mapping of string to any Parameter names mapped to their values. set_params( params) Set the parameters of this estimator. The method works on simple estimators as well as on nested objects (such as pipelines). The latter have parameters of the form <component>__<parameter> so that it's possible to update each component of a nested object. Returns self transform(X, y=None) Return a slice of the input array. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] (default: None) Returns X_slice : shape = [n_samples, k_features] Subset of the feature space where k_features <= n_features","title":"Methods"},{"location":"user_guide/feature_selection/ExhaustiveFeatureSelector/","text":"ExhaustiveFeatureSelector: Optimal feature sets by considering all possible feature combinations Implementation of an exhaustive feature selector for sampling and evaluating all possible feature combinations in a specified range. from mlxtend.feature_selection import ExhaustiveFeatureSelector Overview This exhaustive feature selection algorithm is a wrapper approach for brute-force evaluation of feature subsets; the best subset is selected by optimizing a specified performance metric given an arbitrary regressor or classifier. For instance, if the classifier is a logistic regression and the dataset consists of 4 features, the alogorithm will evaluate all 15 feature combinations (if min_features=1 and max_features=4 ) {0} {1} {2} {3} {0, 1} {0, 2} {0, 3} {1, 2} {1, 3} {2, 3} {0, 1, 2} {0, 1, 3} {0, 2, 3} {1, 2, 3} {0, 1, 2, 3} and select the one that results in the best performance (e.g., classification accuracy) of the logistic regression classifier. Example 1 - A simple Iris example Initializing a simple classifier from scikit-learn: from sklearn.neighbors import KNeighborsClassifier from sklearn.datasets import load_iris from mlxtend.feature_selection import ExhaustiveFeatureSelector as EFS iris = load_iris() X = iris.data y = iris.target knn = KNeighborsClassifier(n_neighbors=3) efs1 = EFS(knn, min_features=1, max_features=4, scoring='accuracy', print_progress=True, cv=5) efs1 = efs1.fit(X, y) print('Best accuracy score: %.2f' % efs1.best_score_) print('Best subset (indices):', efs1.best_idx_) print('Best subset (corresponding names):', efs1.best_feature_names_) Features: 15/15 Best accuracy score: 0.97 Best subset (indices): (0, 2, 3) Best subset (corresponding names): ('0', '2', '3') Feature Names When working with large datasets, the feature indices might be hard to interpret. In this case, we recommend using pandas DataFrames with distinct column names as input: import pandas as pd df_X = pd.DataFrame(X, columns=[\"Sepal length\", \"Sepal width\", \"Petal length\", \"Petal width\"]) df_X.head() .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } Sepal length Sepal width Petal length Petal width 0 5.1 3.5 1.4 0.2 1 4.9 3.0 1.4 0.2 2 4.7 3.2 1.3 0.2 3 4.6 3.1 1.5 0.2 4 5.0 3.6 1.4 0.2 efs1 = efs1.fit(df_X, y) print('Best accuracy score: %.2f' % efs1.best_score_) print('Best subset (indices):', efs1.best_idx_) print('Best subset (corresponding names):', efs1.best_feature_names_) Features: 15/15 Best accuracy score: 0.97 Best subset (indices): (0, 2, 3) Best subset (corresponding names): ('Sepal length', 'Petal length', 'Petal width') Detailed Outputs Via the subsets_ attribute, we can take a look at the selected feature indices at each step: efs1.subsets_ {0: {'feature_idx': (0,), 'cv_scores': array([0.53333333, 0.63333333, 0.7 , 0.8 , 0.56666667]), 'avg_score': 0.6466666666666667, 'feature_names': ('Sepal length',)}, 1: {'feature_idx': (1,), 'cv_scores': array([0.43333333, 0.63333333, 0.53333333, 0.43333333, 0.5 ]), 'avg_score': 0.5066666666666666, 'feature_names': ('Sepal width',)}, 2: {'feature_idx': (2,), 'cv_scores': array([0.93333333, 0.93333333, 0.9 , 0.93333333, 1. ]), 'avg_score': 0.9400000000000001, 'feature_names': ('Petal length',)}, 3: {'feature_idx': (3,), 'cv_scores': array([0.96666667, 0.96666667, 0.93333333, 0.93333333, 1. ]), 'avg_score': 0.96, 'feature_names': ('Petal width',)}, 4: {'feature_idx': (0, 1), 'cv_scores': array([0.66666667, 0.8 , 0.7 , 0.86666667, 0.66666667]), 'avg_score': 0.74, 'feature_names': ('Sepal length', 'Sepal width')}, 5: {'feature_idx': (0, 2), 'cv_scores': array([0.96666667, 1. , 0.86666667, 0.93333333, 0.96666667]), 'avg_score': 0.9466666666666667, 'feature_names': ('Sepal length', 'Petal length')}, 6: {'feature_idx': (0, 3), 'cv_scores': array([0.96666667, 0.96666667, 0.9 , 0.93333333, 1. ]), 'avg_score': 0.9533333333333334, 'feature_names': ('Sepal length', 'Petal width')}, 7: {'feature_idx': (1, 2), 'cv_scores': array([0.93333333, 0.93333333, 0.9 , 0.93333333, 0.93333333]), 'avg_score': 0.9266666666666667, 'feature_names': ('Sepal width', 'Petal length')}, 8: {'feature_idx': (1, 3), 'cv_scores': array([0.96666667, 0.96666667, 0.86666667, 0.93333333, 0.96666667]), 'avg_score': 0.9400000000000001, 'feature_names': ('Sepal width', 'Petal width')}, 9: {'feature_idx': (2, 3), 'cv_scores': array([0.96666667, 0.96666667, 0.9 , 0.93333333, 1. ]), 'avg_score': 0.9533333333333334, 'feature_names': ('Petal length', 'Petal width')}, 10: {'feature_idx': (0, 1, 2), 'cv_scores': array([0.96666667, 0.96666667, 0.86666667, 0.93333333, 0.96666667]), 'avg_score': 0.9400000000000001, 'feature_names': ('Sepal length', 'Sepal width', 'Petal length')}, 11: {'feature_idx': (0, 1, 3), 'cv_scores': array([0.93333333, 0.96666667, 0.9 , 0.93333333, 1. ]), 'avg_score': 0.9466666666666667, 'feature_names': ('Sepal length', 'Sepal width', 'Petal width')}, 12: {'feature_idx': (0, 2, 3), 'cv_scores': array([0.96666667, 0.96666667, 0.96666667, 0.96666667, 1. ]), 'avg_score': 0.9733333333333334, 'feature_names': ('Sepal length', 'Petal length', 'Petal width')}, 13: {'feature_idx': (1, 2, 3), 'cv_scores': array([0.96666667, 0.96666667, 0.93333333, 0.93333333, 1. ]), 'avg_score': 0.96, 'feature_names': ('Sepal width', 'Petal length', 'Petal width')}, 14: {'feature_idx': (0, 1, 2, 3), 'cv_scores': array([0.96666667, 0.96666667, 0.93333333, 0.96666667, 1. ]), 'avg_score': 0.9666666666666668, 'feature_names': ('Sepal length', 'Sepal width', 'Petal length', 'Petal width')}} Example 2 - Visualizing the feature selection results For our convenience, we can visualize the output from the feature selection in a pandas DataFrame format using the get_metric_dict method of the ExhaustiveFeatureSelector object. The columns std_dev and std_err represent the standard deviation and standard errors of the cross-validation scores, respectively. Below, we see the DataFrame of the Sequential Forward Selector from Example 2: import pandas as pd iris = load_iris() X = iris.data y = iris.target knn = KNeighborsClassifier(n_neighbors=3) efs1 = EFS(knn, min_features=1, max_features=4, scoring='accuracy', print_progress=True, cv=5) feature_names = ('sepal length', 'sepal width', 'petal length', 'petal width') df_X = pd.DataFrame( X, columns=[\"Sepal length\", \"Sepal width\", \"Petal length\", \"Petal width\"]) efs1 = efs1.fit(df_X, y) df = pd.DataFrame.from_dict(efs1.get_metric_dict()).T df.sort_values('avg_score', inplace=True, ascending=False) df Features: 15/15 .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } feature_idx cv_scores avg_score feature_names ci_bound std_dev std_err 12 (0, 2, 3) [0.9666666666666667, 0.9666666666666667, 0.966... 0.973333 (Sepal length, Petal length, Petal width) 0.017137 0.013333 0.006667 14 (0, 1, 2, 3) [0.9666666666666667, 0.9666666666666667, 0.933... 0.966667 (Sepal length, Sepal width, Petal length, Peta... 0.027096 0.021082 0.010541 3 (3,) [0.9666666666666667, 0.9666666666666667, 0.933... 0.96 (Petal width,) 0.032061 0.024944 0.012472 13 (1, 2, 3) [0.9666666666666667, 0.9666666666666667, 0.933... 0.96 (Sepal width, Petal length, Petal width) 0.032061 0.024944 0.012472 6 (0, 3) [0.9666666666666667, 0.9666666666666667, 0.9, ... 0.953333 (Sepal length, Petal width) 0.043691 0.033993 0.016997 9 (2, 3) [0.9666666666666667, 0.9666666666666667, 0.9, ... 0.953333 (Petal length, Petal width) 0.043691 0.033993 0.016997 5 (0, 2) [0.9666666666666667, 1.0, 0.8666666666666667, ... 0.946667 (Sepal length, Petal length) 0.058115 0.045216 0.022608 11 (0, 1, 3) [0.9333333333333333, 0.9666666666666667, 0.9, ... 0.946667 (Sepal length, Sepal width, Petal width) 0.043691 0.033993 0.016997 2 (2,) [0.9333333333333333, 0.9333333333333333, 0.9, ... 0.94 (Petal length,) 0.041977 0.03266 0.01633 8 (1, 3) [0.9666666666666667, 0.9666666666666667, 0.866... 0.94 (Sepal width, Petal width) 0.049963 0.038873 0.019437 10 (0, 1, 2) [0.9666666666666667, 0.9666666666666667, 0.866... 0.94 (Sepal length, Sepal width, Petal length) 0.049963 0.038873 0.019437 7 (1, 2) [0.9333333333333333, 0.9333333333333333, 0.9, ... 0.926667 (Sepal width, Petal length) 0.017137 0.013333 0.006667 4 (0, 1) [0.6666666666666666, 0.8, 0.7, 0.8666666666666... 0.74 (Sepal length, Sepal width) 0.102823 0.08 0.04 0 (0,) [0.5333333333333333, 0.6333333333333333, 0.7, ... 0.646667 (Sepal length,) 0.122983 0.095685 0.047842 1 (1,) [0.43333333333333335, 0.6333333333333333, 0.53... 0.506667 (Sepal width,) 0.095416 0.074237 0.037118 import matplotlib.pyplot as plt metric_dict = efs1.get_metric_dict() fig = plt.figure() k_feat = sorted(metric_dict.keys()) avg = [metric_dict[k]['avg_score'] for k in k_feat] upper, lower = [], [] for k in k_feat: upper.append(metric_dict[k]['avg_score'] + metric_dict[k]['std_dev']) lower.append(metric_dict[k]['avg_score'] - metric_dict[k]['std_dev']) plt.fill_between(k_feat, upper, lower, alpha=0.2, color='blue', lw=1) plt.plot(k_feat, avg, color='blue', marker='o') plt.ylabel('Accuracy +/- Standard Deviation') plt.xlabel('Number of Features') feature_min = len(metric_dict[k_feat[0]]['feature_idx']) feature_max = len(metric_dict[k_feat[-1]]['feature_idx']) plt.xticks(k_feat, [str(metric_dict[k]['feature_names']) for k in k_feat], rotation=90) plt.show() Example 3 - Exhaustive feature selection for regression analysis Similar to the classification examples above, the SequentialFeatureSelector also supports scikit-learn's estimators for regression. from sklearn.linear_model import LinearRegression from sklearn.datasets import load_boston boston = load_boston() X, y = boston.data, boston.target lr = LinearRegression() efs = EFS(lr, min_features=10, max_features=12, scoring='neg_mean_squared_error', cv=10) efs.fit(X, y) print('Best MSE score: %.2f' % efs.best_score_ * (-1)) print('Best subset:', efs.best_idx_) /Users/sebastianraschka/miniforge3/lib/python3.9/site-packages/sklearn/utils/deprecation.py:87: FutureWarning: Function load_boston is deprecated; `load_boston` is deprecated in 1.0 and will be removed in 1.2. The Boston housing prices dataset has an ethical problem. You can refer to the documentation of this function for further details. The scikit-learn maintainers therefore strongly discourage the use of this dataset unless the purpose of the code is to study and educate about ethical issues in data science and machine learning. In this special case, you can fetch the dataset from the original source:: import pandas as pd import numpy as np data_url = \"https://lib.stat.cmu.edu/datasets/boston\" raw_df = pd.read_csv(data_url, sep=\"\\s+\", skiprows=22, header=None) data = np.hstack([raw_df.values[::2, :], raw_df.values[1::2, :2]]) target = raw_df.values[1::2, 2] Alternative datasets include the California housing dataset (i.e. :func:`~sklearn.datasets.fetch_california_housing`) and the Ames housing dataset. You can load the datasets as follows:: from sklearn.datasets import fetch_california_housing housing = fetch_california_housing() for the California housing dataset and:: from sklearn.datasets import fetch_openml housing = fetch_openml(name=\"house_prices\", as_frame=True) for the Ames housing dataset. warnings.warn(msg, category=FutureWarning) Features: 377/377 Best subset: (0, 1, 4, 6, 7, 8, 9, 10, 11, 12) Example 4 - Regression and adjusted R2 As shown in Example 3, the exhaustive feature selector can be used for selecting features via a regression model. In regression analysis, there exists the common phenomenon that the R^2 score can become spuriously inflated the more features we choose. Hence, and this is especially true for feature selection, it is useful to make model comparisons based on the adjusted R^2 value rather than the regular R^2 . The adjusted R^2 , \\bar{R}^{2} , accounts for the number of features and examples as follows: \\bar{R}^{2}=1-\\left(1-R^{2}\\right) \\frac{n-1}{n-p-1}, where n is the number of examples and p is the number of features. One of the advantages of scikit-learn's API is that it's consistent, intuitive, and simple to use. However, one downside of this API design is that it can be a bit restrictive for certain scenarios. For instance, scikit-learn scoring function only take two inputs, the predicted and the true target values. Hence, we cannot use scikit-learn's scoring API to compute the adjusted R^2 , which also requires the number of features. However, as a workaround, we can compute the R^2 for the different feature subsets and then do a posthoc computation to obtain the adjusted R^2 . Step 1: Compute R^2 : from sklearn.linear_model import LinearRegression from sklearn.datasets import load_boston boston = load_boston() X, y = boston.data, boston.target lr = LinearRegression() efs = EFS(lr, min_features=10, max_features=12, scoring='r2', cv=10) efs.fit(X, y) print('Best R2 score: %.2f' % efs.best_score_ * (-1)) print('Best subset:', efs.best_idx_) /Users/sebastianraschka/miniforge3/lib/python3.9/site-packages/sklearn/utils/deprecation.py:87: FutureWarning: Function load_boston is deprecated; `load_boston` is deprecated in 1.0 and will be removed in 1.2. The Boston housing prices dataset has an ethical problem. You can refer to the documentation of this function for further details. The scikit-learn maintainers therefore strongly discourage the use of this dataset unless the purpose of the code is to study and educate about ethical issues in data science and machine learning. In this special case, you can fetch the dataset from the original source:: import pandas as pd import numpy as np data_url = \"https://lib.stat.cmu.edu/datasets/boston\" raw_df = pd.read_csv(data_url, sep=\"\\s+\", skiprows=22, header=None) data = np.hstack([raw_df.values[::2, :], raw_df.values[1::2, :2]]) target = raw_df.values[1::2, 2] Alternative datasets include the California housing dataset (i.e. :func:`~sklearn.datasets.fetch_california_housing`) and the Ames housing dataset. You can load the datasets as follows:: from sklearn.datasets import fetch_california_housing housing = fetch_california_housing() for the California housing dataset and:: from sklearn.datasets import fetch_openml housing = fetch_openml(name=\"house_prices\", as_frame=True) for the Ames housing dataset. warnings.warn(msg, category=FutureWarning) Features: 377/377 Best subset: (1, 3, 5, 6, 7, 8, 9, 10, 11, 12) Step 2: Compute adjusted R^2 : def adjust_r2(r2, num_examples, num_features): coef = (num_examples - 1) / (num_examples - num_features - 1) return 1 - (1 - r2) * coef for i in efs.subsets_: efs.subsets_[i]['adjusted_avg_score'] = ( adjust_r2(r2=efs.subsets_[i]['avg_score'], num_examples=X.shape[0]/10, num_features=len(efs.subsets_[i]['feature_idx'])) ) Step 3: Select best subset based on adjusted R^2 : score = -99e10 for i in efs.subsets_: score = efs.subsets_[i]['adjusted_avg_score'] if ( efs.subsets_[i]['adjusted_avg_score'] == score and len(efs.subsets_[i]['feature_idx']) < len(efs.best_idx_) )\\ or efs.subsets_[i]['adjusted_avg_score'] > score: efs.best_idx_ = efs.subsets_[i]['feature_idx'] print('Best adjusted R2 score: %.2f' % efs.best_score_ * (-1)) print('Best subset:', efs.best_idx_) Best subset: (1, 3, 5, 6, 7, 8, 9, 10, 11, 12) Example 5 - Using the selected feature subset For making new predictions # Initialize the dataset from sklearn.neighbors import KNeighborsClassifier from sklearn.datasets import load_iris from sklearn.model_selection import train_test_split iris = load_iris() X, y = iris.data, iris.target X_train, X_test, y_train, y_test = train_test_split( X, y, test_size=0.33, random_state=1) knn = KNeighborsClassifier(n_neighbors=3) # Select the \"best\" three features via # 5-fold cross-validation on the training set. from mlxtend.feature_selection import ExhaustiveFeatureSelector as EFS efs1 = EFS(knn, min_features=1, max_features=4, scoring='accuracy', cv=5) efs1 = efs1.fit(X_train, y_train) Features: 15/15 print('Selected features:', efs1.best_idx_) Selected features: (2, 3) # Generate the new subsets based on the selected features # Note that the transform call is equivalent to # X_train[:, efs1.k_feature_idx_] X_train_efs = efs1.transform(X_train) X_test_efs = efs1.transform(X_test) # Fit the estimator using the new feature subset # and make a prediction on the test data knn.fit(X_train_efs, y_train) y_pred = knn.predict(X_test_efs) # Compute the accuracy of the prediction acc = float((y_test == y_pred).sum()) / y_pred.shape[0] print('Test set accuracy: %.2f %%' % (acc*100)) Test set accuracy: 96.00 % Example 6 - Exhaustive feature selection and GridSearch # Initialize the dataset from sklearn.datasets import load_iris from sklearn.model_selection import train_test_split iris = load_iris() X, y = iris.data, iris.target X_train, X_test, y_train, y_test = train_test_split( X, y, test_size=0.33, random_state=1) Use scikit-learn's GridSearch to tune the hyperparameters of the LogisticRegression estimator inside the ExhaustiveFeatureSelector and use it for prediction in the pipeline. Note that the clone_estimator attribute needs to be set to False . from sklearn.model_selection import GridSearchCV from sklearn.pipeline import make_pipeline from sklearn.linear_model import LogisticRegression from mlxtend.feature_selection import ExhaustiveFeatureSelector as EFS lr = LogisticRegression(multi_class='multinomial', solver='newton-cg', random_state=123) efs1 = EFS(estimator=lr, min_features=2, max_features=3, scoring='accuracy', print_progress=False, clone_estimator=False, cv=5, n_jobs=1) pipe = make_pipeline(efs1, lr) param_grid = {'exhaustivefeatureselector__estimator__C': [0.1, 1.0, 10.0]} gs = GridSearchCV(estimator=pipe, param_grid=param_grid, scoring='accuracy', n_jobs=1, cv=2, verbose=1, refit=False) # run gridearch gs = gs.fit(X_train, y_train) Fitting 2 folds for each of 3 candidates, totalling 6 fits ... and the \"best\" parameters determined by GridSearch are ... print(\"Best parameters via GridSearch\", gs.best_params_) Best parameters via GridSearch {'exhaustivefeatureselector__estimator__C': 0.1} Obtaining the best k feature indices after GridSearch If we are interested in the best k best feature indices via SequentialFeatureSelection.best_idx_ , we have to initialize a GridSearchCV object with refit=True . Now, the grid search object will take the complete training dataset and the best parameters, which it found via cross-validation, to train the estimator pipeline. gs = GridSearchCV(estimator=pipe, param_grid=param_grid, scoring='accuracy', n_jobs=1, cv=2, verbose=1, refit=True) After running the grid search, we can access the individual pipeline objects of the best_estimator_ via the steps attribute. gs = gs.fit(X_train, y_train) gs.best_estimator_.steps Fitting 2 folds for each of 3 candidates, totalling 6 fits [('exhaustivefeatureselector', ExhaustiveFeatureSelector(clone_estimator=False, estimator=LogisticRegression(C=0.1, multi_class='multinomial', random_state=123, solver='newton-cg'), feature_groups=[[0], [1], [2], [3]], max_features=3, min_features=2, print_progress=False)), ('logisticregression', LogisticRegression(multi_class='multinomial', random_state=123, solver='newton-cg'))] Via sub-indexing, we can then obtain the best-selected feature subset: print('Best features:', gs.best_estimator_.steps[0][1].best_idx_) Best features: (2, 3) During cross-validation, this feature combination had a CV accuracy of: print('Best score:', gs.best_score_) Best score: 0.96 gs.best_params_ {'exhaustivefeatureselector__estimator__C': 0.1} Alternatively , if we can set the \"best grid search parameters\" in our pipeline manually if we ran GridSearchCV with refit=False . It should yield the same results: pipe.set_params(**gs.best_params_).fit(X_train, y_train) print('Best features:', pipe.steps[0][1].best_idx_) Best features: (2, 3) Example 7 - Exhaustive Feature Selection with LOOCV The ExhaustiveFeatureSelector is not restricted to k-fold cross-validation. You can use any type of cross-validation method that supports the general scikit-learn cross-validation API. The following example illustrates the use of scikit-learn's LeaveOneOut cross-validation method in combination with the exhaustive feature selector. from sklearn.neighbors import KNeighborsClassifier from sklearn.datasets import load_iris from mlxtend.feature_selection import ExhaustiveFeatureSelector as EFS from sklearn.model_selection import LeaveOneOut iris = load_iris() X = iris.data y = iris.target knn = KNeighborsClassifier(n_neighbors=3) efs1 = EFS(knn, min_features=1, max_features=4, scoring='accuracy', print_progress=True, cv=LeaveOneOut()) ### Use cross-validation generator here efs1 = efs1.fit(X, y) print('Best accuracy score: %.2f' % efs1.best_score_) print('Best subset (indices):', efs1.best_idx_) print('Best subset (corresponding names):', efs1.best_feature_names_) Features: 15/15 Best accuracy score: 0.96 Best subset (indices): (3,) Best subset (corresponding names): ('3',) Example 8 - Interrupting Long Runs for Intermediate Results If your run is taking too long, it is possible to trigger a KeyboardInterrupt (e.g., ctrl+c on a Mac, or interrupting the cell in a Jupyter notebook) to obtain temporary results. Toy dataset from sklearn.datasets import make_classification from sklearn.model_selection import train_test_split X, y = make_classification( n_samples=200000, n_features=6, n_informative=2, n_redundant=1, n_repeated=1, n_clusters_per_class=2, flip_y=0.05, class_sep=0.5, random_state=123, ) X_train, X_test, y_train, y_test = train_test_split( X, y, test_size=0.2, random_state=123 ) Long run with interruption from mlxtend.feature_selection import ExhaustiveFeatureSelector as EFS from sklearn.linear_model import LogisticRegression model = LogisticRegression(max_iter=10000) efs1 = EFS(model, min_features=1, max_features=4, print_progress=True, scoring='accuracy') efs1 = efs1.fit(X_train, y_train) Features: 56/56 Finalizing the fit Note that the feature selection run hasn't finished, so certain attributes may not be available. In order to use the EFS instance, it is recommended to call finalize_fit , which will make EFS estimator appear as \"fitted\" process the temporary results: efs1.finalize_fit() print('Best accuracy score: %.2f' % efs1.best_score_) print('Best subset (indices):', efs1.best_idx_) Best accuracy score: 0.73 Best subset (indices): (1, 2) Example 9 - Working with Feature Groups Since mlxtend v0.21.0, it is possible to specify feature groups. Feature groups allow you to group certain features together, such that they are always selected as a group. This can be very useful in contexts similar to one-hot encoding -- if you want to treat the one-hot encoded feature as a single feature: In the following example, we specify sepal length and sepal width as a feature group so that they are always selected together: from sklearn.datasets import load_iris import pandas as pd iris = load_iris() X = iris.data y = iris.target X_df = pd.DataFrame(X, columns=['sepal len', 'petal len', 'sepal wid', 'petal wid']) X_df.head() .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } sepal len petal len sepal wid petal wid 0 5.1 3.5 1.4 0.2 1 4.9 3.0 1.4 0.2 2 4.7 3.2 1.3 0.2 3 4.6 3.1 1.5 0.2 4 5.0 3.6 1.4 0.2 from sklearn.neighbors import KNeighborsClassifier from mlxtend.feature_selection import ExhaustiveFeatureSelector as EFS knn = KNeighborsClassifier(n_neighbors=3) efs1 = EFS(knn, min_features=2, max_features=2, scoring='accuracy', feature_groups=[['sepal len', 'sepal wid'], ['petal len'], ['petal wid']], cv=3) efs1 = efs1.fit(X_df, y) print('Best accuracy score: %.2f' % efs1.best_score_) print('Best subset (indices):', efs1.best_idx_) print('Best subset (corresponding names):', efs1.best_feature_names_) Features: 3/3 Best accuracy score: 0.97 Best subset (indices): (0, 2, 3) Best subset (corresponding names): ('sepal len', 'sepal wid', 'petal wid') Notice that the returned number of features is 3, since the number of min_features and max_features corresponds to the number of feature groups. I.e., we have 2 feature groups in ['sepal len', 'sepal wid'], ['petal wid'] , but it expands to 3 features. API ExhaustiveFeatureSelector(estimator, min_features=1, max_features=1, print_progress=True, scoring='accuracy', cv=5, n_jobs=1, pre_dispatch='2 n_jobs', clone_estimator=True, fixed_features=None, feature_groups=None)* Exhaustive Feature Selection for Classification and Regression. (new in v0.4.3) Parameters estimator : scikit-learn classifier or regressor min_features : int (default: 1) Minumum number of features to select max_features : int (default: 1) Maximum number of features to select. If parameter feature_groups is not None, the number of features is equal to the number of feature groups, i.e. len(feature_groups) . For example, if feature_groups = [[0], [1], [2, 3], [4]] , then the max_features value cannot exceed 4. print_progress : bool (default: True) Prints progress as the number of epochs to stderr. scoring : str, (default='accuracy') Scoring metric in {accuracy, f1, precision, recall, roc_auc} for classifiers, {'mean_absolute_error', 'mean_squared_error', 'median_absolute_error', 'r2'} for regressors, or a callable object or function with signature scorer(estimator, X, y) . cv : int (default: 5) Scikit-learn cross-validation generator or int . If estimator is a classifier (or y consists of integer class labels), stratified k-fold is performed, and regular k-fold cross-validation otherwise. No cross-validation if cv is None, False, or 0. n_jobs : int (default: 1) The number of CPUs to use for evaluating different feature subsets in parallel. -1 means 'all CPUs'. pre_dispatch : int, or string (default: '2*n_jobs') Controls the number of jobs that get dispatched during parallel execution if n_jobs > 1 or n_jobs=-1 . Reducing this number can be useful to avoid an explosion of memory consumption when more jobs get dispatched than CPUs can process. This parameter can be: None, in which case all the jobs are immediately created and spawned. Use this for lightweight and fast-running jobs, to avoid delays due to on-demand spawning of the jobs An int, giving the exact number of total jobs that are spawned A string, giving an expression as a function of n_jobs, as in 2*n_jobs clone_estimator : bool (default: True) Clones estimator if True; works with the original estimator instance if False. Set to False if the estimator doesn't implement scikit-learn's set_params and get_params methods. In addition, it is required to set cv=0, and n_jobs=1. fixed_features : tuple (default: None) If not None , the feature indices provided as a tuple will be regarded as fixed by the feature selector. For example, if fixed_features=(1, 3, 7) , the 2nd, 4th, and 8th feature are guaranteed to be present in the solution. Note that if fixed_features is not None , make sure that the number of features to be selected is greater than len(fixed_features) . In other words, ensure that k_features > len(fixed_features) . feature_groups : list or None (default: None) Optional argument for treating certain features as a group. This means, the features within a group are always selected together, never split. For example, feature_groups=[[1], [2], [3, 4, 5]] specifies 3 feature groups.In this case, possible feature selection results with k_features=2 are [[1], [2] , [[1], [3, 4, 5]] , or [[2], [3, 4, 5]] . Feature groups can be useful for interpretability, for example, if features 3, 4, 5 are one-hot encoded features. (For more details, please read the notes at the bottom of this docstring). New in mlxtend v. 0.21.0. Attributes best_idx_ : array-like, shape = [n_predictions] Feature Indices of the selected feature subsets. best_feature_names_ : array-like, shape = [n_predictions] Feature names of the selected feature subsets. If pandas DataFrames are used in the fit method, the feature names correspond to the column names. Otherwise, the feature names are string representation of the feature array indices. New in v 0.13.0. best_score_ : float Cross validation average score of the selected subset. subsets_ : dict A dictionary of selected feature subsets during the exhaustive selection, where the dictionary keys are the lengths k of these feature subsets. The dictionary values are dictionaries themselves with the following keys: 'feature_idx' (tuple of indices of the feature subset) 'feature_names' (tuple of feature names of the feat. subset) 'cv_scores' (list individual cross-validation scores) 'avg_score' (average cross-validation score) Note that if pandas DataFrames are used in the fit method, the 'feature_names' correspond to the column names. Otherwise, the feature names are string representation of the feature array indices. The 'feature_names' is new in v. 0.13.0. Notes (1) If parameter feature_groups is not None, the number of features is equal to the number of feature groups, i.e. len(feature_groups) . For example, if feature_groups = [[0], [1], [2, 3], [4]] , then the max_features value cannot exceed 4. (2) Although two or more individual features may be considered as one group throughout the feature-selection process, it does not mean the individual features of that group have the same impact on the outcome. For instance, in linear regression, the coefficient of the feature 2 and 3 can be different even if they are considered as one group in feature_groups. (3) If both fixed_features and feature_groups are specified, ensure that each feature group contains the fixed_features selection. E.g., for a 3-feature set fixed_features=[0, 1] and feature_groups=[[0, 1], [2]] is valid; fixed_features=[0, 1] and feature_groups=[[0], [1, 2]] is not valid. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/feature_selection/ExhaustiveFeatureSelector/ Methods fit(X, y, groups=None, fit_params) Perform feature selection and learn model from training data. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. New in v 0.13.0: pandas DataFrames are now also accepted as argument for X. y : array-like, shape = [n_samples] Target values. groups : array-like, with shape (n_samples,), optional Group labels for the samples used while splitting the dataset into train/test set. Passed to the fit method of the cross-validator. fit_params : dict of string -> object, optional Parameters to pass to to the fit method of classifier. Returns self : object fit_transform(X, y, groups=None, fit_params) Fit to training data and return the best selected features from X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. New in v 0.13.0: pandas DataFrames are now also accepted as argument for X. y : array-like, shape = [n_samples] Target values. groups : array-like, with shape (n_samples,), optional Group labels for the samples used while splitting the dataset into train/test set. Passed to the fit method of the cross-validator. fit_params : dict of string -> object, optional Parameters to pass to to the fit method of classifier. Returns Feature subset of X, shape={n_samples, k_features} get_metric_dict(confidence_interval=0.95) Return metric dictionary Parameters confidence_interval : float (default: 0.95) A positive float between 0.0 and 1.0 to compute the confidence interval bounds of the CV score averages. Returns Dictionary with items where each dictionary value is a list with the number of iterations (number of feature subsets) as its length. The dictionary keys corresponding to these lists are as follows: 'feature_idx': tuple of the indices of the feature subset 'cv_scores': list with individual CV scores 'avg_score': of CV average scores 'std_dev': standard deviation of the CV score average 'std_err': standard error of the CV score average 'ci_bound': confidence interval bound of the CV score average get_params(deep=True) Get parameters for this estimator. Parameters deep : bool, default=True If True, will return the parameters for this estimator and contained subobjects that are estimators. Returns params : dict Parameter names mapped to their values. set_params( params) Set the parameters of this estimator. The method works on simple estimators as well as on nested objects (such as :class:`~sklearn.pipeline.Pipeline`). The latter have parameters of the form ``<component>__<parameter>`` so that it's possible to update each component of a nested object. Parameters **params : dict Estimator parameters. Returns self : estimator instance Estimator instance. transform(X) Return the best selected features from X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. New in v 0.13.0: pandas DataFrames are now also accepted as argument for X. Returns Feature subset of X, shape={n_samples, k_features}","title":"ExhaustiveFeatureSelector: Optimal feature sets by considering all possible feature combinations"},{"location":"user_guide/feature_selection/ExhaustiveFeatureSelector/#exhaustivefeatureselector-optimal-feature-sets-by-considering-all-possible-feature-combinations","text":"Implementation of an exhaustive feature selector for sampling and evaluating all possible feature combinations in a specified range. from mlxtend.feature_selection import ExhaustiveFeatureSelector","title":"ExhaustiveFeatureSelector: Optimal feature sets by considering all possible feature combinations"},{"location":"user_guide/feature_selection/ExhaustiveFeatureSelector/#overview","text":"This exhaustive feature selection algorithm is a wrapper approach for brute-force evaluation of feature subsets; the best subset is selected by optimizing a specified performance metric given an arbitrary regressor or classifier. For instance, if the classifier is a logistic regression and the dataset consists of 4 features, the alogorithm will evaluate all 15 feature combinations (if min_features=1 and max_features=4 ) {0} {1} {2} {3} {0, 1} {0, 2} {0, 3} {1, 2} {1, 3} {2, 3} {0, 1, 2} {0, 1, 3} {0, 2, 3} {1, 2, 3} {0, 1, 2, 3} and select the one that results in the best performance (e.g., classification accuracy) of the logistic regression classifier.","title":"Overview"},{"location":"user_guide/feature_selection/ExhaustiveFeatureSelector/#example-1-a-simple-iris-example","text":"Initializing a simple classifier from scikit-learn: from sklearn.neighbors import KNeighborsClassifier from sklearn.datasets import load_iris from mlxtend.feature_selection import ExhaustiveFeatureSelector as EFS iris = load_iris() X = iris.data y = iris.target knn = KNeighborsClassifier(n_neighbors=3) efs1 = EFS(knn, min_features=1, max_features=4, scoring='accuracy', print_progress=True, cv=5) efs1 = efs1.fit(X, y) print('Best accuracy score: %.2f' % efs1.best_score_) print('Best subset (indices):', efs1.best_idx_) print('Best subset (corresponding names):', efs1.best_feature_names_) Features: 15/15 Best accuracy score: 0.97 Best subset (indices): (0, 2, 3) Best subset (corresponding names): ('0', '2', '3')","title":"Example 1 - A simple Iris example"},{"location":"user_guide/feature_selection/ExhaustiveFeatureSelector/#feature-names","text":"When working with large datasets, the feature indices might be hard to interpret. In this case, we recommend using pandas DataFrames with distinct column names as input: import pandas as pd df_X = pd.DataFrame(X, columns=[\"Sepal length\", \"Sepal width\", \"Petal length\", \"Petal width\"]) df_X.head() .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } Sepal length Sepal width Petal length Petal width 0 5.1 3.5 1.4 0.2 1 4.9 3.0 1.4 0.2 2 4.7 3.2 1.3 0.2 3 4.6 3.1 1.5 0.2 4 5.0 3.6 1.4 0.2 efs1 = efs1.fit(df_X, y) print('Best accuracy score: %.2f' % efs1.best_score_) print('Best subset (indices):', efs1.best_idx_) print('Best subset (corresponding names):', efs1.best_feature_names_) Features: 15/15 Best accuracy score: 0.97 Best subset (indices): (0, 2, 3) Best subset (corresponding names): ('Sepal length', 'Petal length', 'Petal width')","title":"Feature Names"},{"location":"user_guide/feature_selection/ExhaustiveFeatureSelector/#detailed-outputs","text":"Via the subsets_ attribute, we can take a look at the selected feature indices at each step: efs1.subsets_ {0: {'feature_idx': (0,), 'cv_scores': array([0.53333333, 0.63333333, 0.7 , 0.8 , 0.56666667]), 'avg_score': 0.6466666666666667, 'feature_names': ('Sepal length',)}, 1: {'feature_idx': (1,), 'cv_scores': array([0.43333333, 0.63333333, 0.53333333, 0.43333333, 0.5 ]), 'avg_score': 0.5066666666666666, 'feature_names': ('Sepal width',)}, 2: {'feature_idx': (2,), 'cv_scores': array([0.93333333, 0.93333333, 0.9 , 0.93333333, 1. ]), 'avg_score': 0.9400000000000001, 'feature_names': ('Petal length',)}, 3: {'feature_idx': (3,), 'cv_scores': array([0.96666667, 0.96666667, 0.93333333, 0.93333333, 1. ]), 'avg_score': 0.96, 'feature_names': ('Petal width',)}, 4: {'feature_idx': (0, 1), 'cv_scores': array([0.66666667, 0.8 , 0.7 , 0.86666667, 0.66666667]), 'avg_score': 0.74, 'feature_names': ('Sepal length', 'Sepal width')}, 5: {'feature_idx': (0, 2), 'cv_scores': array([0.96666667, 1. , 0.86666667, 0.93333333, 0.96666667]), 'avg_score': 0.9466666666666667, 'feature_names': ('Sepal length', 'Petal length')}, 6: {'feature_idx': (0, 3), 'cv_scores': array([0.96666667, 0.96666667, 0.9 , 0.93333333, 1. ]), 'avg_score': 0.9533333333333334, 'feature_names': ('Sepal length', 'Petal width')}, 7: {'feature_idx': (1, 2), 'cv_scores': array([0.93333333, 0.93333333, 0.9 , 0.93333333, 0.93333333]), 'avg_score': 0.9266666666666667, 'feature_names': ('Sepal width', 'Petal length')}, 8: {'feature_idx': (1, 3), 'cv_scores': array([0.96666667, 0.96666667, 0.86666667, 0.93333333, 0.96666667]), 'avg_score': 0.9400000000000001, 'feature_names': ('Sepal width', 'Petal width')}, 9: {'feature_idx': (2, 3), 'cv_scores': array([0.96666667, 0.96666667, 0.9 , 0.93333333, 1. ]), 'avg_score': 0.9533333333333334, 'feature_names': ('Petal length', 'Petal width')}, 10: {'feature_idx': (0, 1, 2), 'cv_scores': array([0.96666667, 0.96666667, 0.86666667, 0.93333333, 0.96666667]), 'avg_score': 0.9400000000000001, 'feature_names': ('Sepal length', 'Sepal width', 'Petal length')}, 11: {'feature_idx': (0, 1, 3), 'cv_scores': array([0.93333333, 0.96666667, 0.9 , 0.93333333, 1. ]), 'avg_score': 0.9466666666666667, 'feature_names': ('Sepal length', 'Sepal width', 'Petal width')}, 12: {'feature_idx': (0, 2, 3), 'cv_scores': array([0.96666667, 0.96666667, 0.96666667, 0.96666667, 1. ]), 'avg_score': 0.9733333333333334, 'feature_names': ('Sepal length', 'Petal length', 'Petal width')}, 13: {'feature_idx': (1, 2, 3), 'cv_scores': array([0.96666667, 0.96666667, 0.93333333, 0.93333333, 1. ]), 'avg_score': 0.96, 'feature_names': ('Sepal width', 'Petal length', 'Petal width')}, 14: {'feature_idx': (0, 1, 2, 3), 'cv_scores': array([0.96666667, 0.96666667, 0.93333333, 0.96666667, 1. ]), 'avg_score': 0.9666666666666668, 'feature_names': ('Sepal length', 'Sepal width', 'Petal length', 'Petal width')}}","title":"Detailed Outputs"},{"location":"user_guide/feature_selection/ExhaustiveFeatureSelector/#example-2-visualizing-the-feature-selection-results","text":"For our convenience, we can visualize the output from the feature selection in a pandas DataFrame format using the get_metric_dict method of the ExhaustiveFeatureSelector object. The columns std_dev and std_err represent the standard deviation and standard errors of the cross-validation scores, respectively. Below, we see the DataFrame of the Sequential Forward Selector from Example 2: import pandas as pd iris = load_iris() X = iris.data y = iris.target knn = KNeighborsClassifier(n_neighbors=3) efs1 = EFS(knn, min_features=1, max_features=4, scoring='accuracy', print_progress=True, cv=5) feature_names = ('sepal length', 'sepal width', 'petal length', 'petal width') df_X = pd.DataFrame( X, columns=[\"Sepal length\", \"Sepal width\", \"Petal length\", \"Petal width\"]) efs1 = efs1.fit(df_X, y) df = pd.DataFrame.from_dict(efs1.get_metric_dict()).T df.sort_values('avg_score', inplace=True, ascending=False) df Features: 15/15 .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } feature_idx cv_scores avg_score feature_names ci_bound std_dev std_err 12 (0, 2, 3) [0.9666666666666667, 0.9666666666666667, 0.966... 0.973333 (Sepal length, Petal length, Petal width) 0.017137 0.013333 0.006667 14 (0, 1, 2, 3) [0.9666666666666667, 0.9666666666666667, 0.933... 0.966667 (Sepal length, Sepal width, Petal length, Peta... 0.027096 0.021082 0.010541 3 (3,) [0.9666666666666667, 0.9666666666666667, 0.933... 0.96 (Petal width,) 0.032061 0.024944 0.012472 13 (1, 2, 3) [0.9666666666666667, 0.9666666666666667, 0.933... 0.96 (Sepal width, Petal length, Petal width) 0.032061 0.024944 0.012472 6 (0, 3) [0.9666666666666667, 0.9666666666666667, 0.9, ... 0.953333 (Sepal length, Petal width) 0.043691 0.033993 0.016997 9 (2, 3) [0.9666666666666667, 0.9666666666666667, 0.9, ... 0.953333 (Petal length, Petal width) 0.043691 0.033993 0.016997 5 (0, 2) [0.9666666666666667, 1.0, 0.8666666666666667, ... 0.946667 (Sepal length, Petal length) 0.058115 0.045216 0.022608 11 (0, 1, 3) [0.9333333333333333, 0.9666666666666667, 0.9, ... 0.946667 (Sepal length, Sepal width, Petal width) 0.043691 0.033993 0.016997 2 (2,) [0.9333333333333333, 0.9333333333333333, 0.9, ... 0.94 (Petal length,) 0.041977 0.03266 0.01633 8 (1, 3) [0.9666666666666667, 0.9666666666666667, 0.866... 0.94 (Sepal width, Petal width) 0.049963 0.038873 0.019437 10 (0, 1, 2) [0.9666666666666667, 0.9666666666666667, 0.866... 0.94 (Sepal length, Sepal width, Petal length) 0.049963 0.038873 0.019437 7 (1, 2) [0.9333333333333333, 0.9333333333333333, 0.9, ... 0.926667 (Sepal width, Petal length) 0.017137 0.013333 0.006667 4 (0, 1) [0.6666666666666666, 0.8, 0.7, 0.8666666666666... 0.74 (Sepal length, Sepal width) 0.102823 0.08 0.04 0 (0,) [0.5333333333333333, 0.6333333333333333, 0.7, ... 0.646667 (Sepal length,) 0.122983 0.095685 0.047842 1 (1,) [0.43333333333333335, 0.6333333333333333, 0.53... 0.506667 (Sepal width,) 0.095416 0.074237 0.037118 import matplotlib.pyplot as plt metric_dict = efs1.get_metric_dict() fig = plt.figure() k_feat = sorted(metric_dict.keys()) avg = [metric_dict[k]['avg_score'] for k in k_feat] upper, lower = [], [] for k in k_feat: upper.append(metric_dict[k]['avg_score'] + metric_dict[k]['std_dev']) lower.append(metric_dict[k]['avg_score'] - metric_dict[k]['std_dev']) plt.fill_between(k_feat, upper, lower, alpha=0.2, color='blue', lw=1) plt.plot(k_feat, avg, color='blue', marker='o') plt.ylabel('Accuracy +/- Standard Deviation') plt.xlabel('Number of Features') feature_min = len(metric_dict[k_feat[0]]['feature_idx']) feature_max = len(metric_dict[k_feat[-1]]['feature_idx']) plt.xticks(k_feat, [str(metric_dict[k]['feature_names']) for k in k_feat], rotation=90) plt.show()","title":"Example 2 - Visualizing the feature selection results"},{"location":"user_guide/feature_selection/ExhaustiveFeatureSelector/#example-3-exhaustive-feature-selection-for-regression-analysis","text":"Similar to the classification examples above, the SequentialFeatureSelector also supports scikit-learn's estimators for regression. from sklearn.linear_model import LinearRegression from sklearn.datasets import load_boston boston = load_boston() X, y = boston.data, boston.target lr = LinearRegression() efs = EFS(lr, min_features=10, max_features=12, scoring='neg_mean_squared_error', cv=10) efs.fit(X, y) print('Best MSE score: %.2f' % efs.best_score_ * (-1)) print('Best subset:', efs.best_idx_) /Users/sebastianraschka/miniforge3/lib/python3.9/site-packages/sklearn/utils/deprecation.py:87: FutureWarning: Function load_boston is deprecated; `load_boston` is deprecated in 1.0 and will be removed in 1.2. The Boston housing prices dataset has an ethical problem. You can refer to the documentation of this function for further details. The scikit-learn maintainers therefore strongly discourage the use of this dataset unless the purpose of the code is to study and educate about ethical issues in data science and machine learning. In this special case, you can fetch the dataset from the original source:: import pandas as pd import numpy as np data_url = \"https://lib.stat.cmu.edu/datasets/boston\" raw_df = pd.read_csv(data_url, sep=\"\\s+\", skiprows=22, header=None) data = np.hstack([raw_df.values[::2, :], raw_df.values[1::2, :2]]) target = raw_df.values[1::2, 2] Alternative datasets include the California housing dataset (i.e. :func:`~sklearn.datasets.fetch_california_housing`) and the Ames housing dataset. You can load the datasets as follows:: from sklearn.datasets import fetch_california_housing housing = fetch_california_housing() for the California housing dataset and:: from sklearn.datasets import fetch_openml housing = fetch_openml(name=\"house_prices\", as_frame=True) for the Ames housing dataset. warnings.warn(msg, category=FutureWarning) Features: 377/377 Best subset: (0, 1, 4, 6, 7, 8, 9, 10, 11, 12)","title":"Example 3 - Exhaustive feature selection for regression analysis"},{"location":"user_guide/feature_selection/ExhaustiveFeatureSelector/#example-4-regression-and-adjusted-r2","text":"As shown in Example 3, the exhaustive feature selector can be used for selecting features via a regression model. In regression analysis, there exists the common phenomenon that the R^2 score can become spuriously inflated the more features we choose. Hence, and this is especially true for feature selection, it is useful to make model comparisons based on the adjusted R^2 value rather than the regular R^2 . The adjusted R^2 , \\bar{R}^{2} , accounts for the number of features and examples as follows: \\bar{R}^{2}=1-\\left(1-R^{2}\\right) \\frac{n-1}{n-p-1}, where n is the number of examples and p is the number of features. One of the advantages of scikit-learn's API is that it's consistent, intuitive, and simple to use. However, one downside of this API design is that it can be a bit restrictive for certain scenarios. For instance, scikit-learn scoring function only take two inputs, the predicted and the true target values. Hence, we cannot use scikit-learn's scoring API to compute the adjusted R^2 , which also requires the number of features. However, as a workaround, we can compute the R^2 for the different feature subsets and then do a posthoc computation to obtain the adjusted R^2 . Step 1: Compute R^2 : from sklearn.linear_model import LinearRegression from sklearn.datasets import load_boston boston = load_boston() X, y = boston.data, boston.target lr = LinearRegression() efs = EFS(lr, min_features=10, max_features=12, scoring='r2', cv=10) efs.fit(X, y) print('Best R2 score: %.2f' % efs.best_score_ * (-1)) print('Best subset:', efs.best_idx_) /Users/sebastianraschka/miniforge3/lib/python3.9/site-packages/sklearn/utils/deprecation.py:87: FutureWarning: Function load_boston is deprecated; `load_boston` is deprecated in 1.0 and will be removed in 1.2. The Boston housing prices dataset has an ethical problem. You can refer to the documentation of this function for further details. The scikit-learn maintainers therefore strongly discourage the use of this dataset unless the purpose of the code is to study and educate about ethical issues in data science and machine learning. In this special case, you can fetch the dataset from the original source:: import pandas as pd import numpy as np data_url = \"https://lib.stat.cmu.edu/datasets/boston\" raw_df = pd.read_csv(data_url, sep=\"\\s+\", skiprows=22, header=None) data = np.hstack([raw_df.values[::2, :], raw_df.values[1::2, :2]]) target = raw_df.values[1::2, 2] Alternative datasets include the California housing dataset (i.e. :func:`~sklearn.datasets.fetch_california_housing`) and the Ames housing dataset. You can load the datasets as follows:: from sklearn.datasets import fetch_california_housing housing = fetch_california_housing() for the California housing dataset and:: from sklearn.datasets import fetch_openml housing = fetch_openml(name=\"house_prices\", as_frame=True) for the Ames housing dataset. warnings.warn(msg, category=FutureWarning) Features: 377/377 Best subset: (1, 3, 5, 6, 7, 8, 9, 10, 11, 12) Step 2: Compute adjusted R^2 : def adjust_r2(r2, num_examples, num_features): coef = (num_examples - 1) / (num_examples - num_features - 1) return 1 - (1 - r2) * coef for i in efs.subsets_: efs.subsets_[i]['adjusted_avg_score'] = ( adjust_r2(r2=efs.subsets_[i]['avg_score'], num_examples=X.shape[0]/10, num_features=len(efs.subsets_[i]['feature_idx'])) ) Step 3: Select best subset based on adjusted R^2 : score = -99e10 for i in efs.subsets_: score = efs.subsets_[i]['adjusted_avg_score'] if ( efs.subsets_[i]['adjusted_avg_score'] == score and len(efs.subsets_[i]['feature_idx']) < len(efs.best_idx_) )\\ or efs.subsets_[i]['adjusted_avg_score'] > score: efs.best_idx_ = efs.subsets_[i]['feature_idx'] print('Best adjusted R2 score: %.2f' % efs.best_score_ * (-1)) print('Best subset:', efs.best_idx_) Best subset: (1, 3, 5, 6, 7, 8, 9, 10, 11, 12)","title":"Example 4 - Regression and adjusted R2"},{"location":"user_guide/feature_selection/ExhaustiveFeatureSelector/#example-5-using-the-selected-feature-subset-for-making-new-predictions","text":"# Initialize the dataset from sklearn.neighbors import KNeighborsClassifier from sklearn.datasets import load_iris from sklearn.model_selection import train_test_split iris = load_iris() X, y = iris.data, iris.target X_train, X_test, y_train, y_test = train_test_split( X, y, test_size=0.33, random_state=1) knn = KNeighborsClassifier(n_neighbors=3) # Select the \"best\" three features via # 5-fold cross-validation on the training set. from mlxtend.feature_selection import ExhaustiveFeatureSelector as EFS efs1 = EFS(knn, min_features=1, max_features=4, scoring='accuracy', cv=5) efs1 = efs1.fit(X_train, y_train) Features: 15/15 print('Selected features:', efs1.best_idx_) Selected features: (2, 3) # Generate the new subsets based on the selected features # Note that the transform call is equivalent to # X_train[:, efs1.k_feature_idx_] X_train_efs = efs1.transform(X_train) X_test_efs = efs1.transform(X_test) # Fit the estimator using the new feature subset # and make a prediction on the test data knn.fit(X_train_efs, y_train) y_pred = knn.predict(X_test_efs) # Compute the accuracy of the prediction acc = float((y_test == y_pred).sum()) / y_pred.shape[0] print('Test set accuracy: %.2f %%' % (acc*100)) Test set accuracy: 96.00 %","title":"Example 5 - Using the selected feature subset For making new predictions"},{"location":"user_guide/feature_selection/ExhaustiveFeatureSelector/#example-6-exhaustive-feature-selection-and-gridsearch","text":"# Initialize the dataset from sklearn.datasets import load_iris from sklearn.model_selection import train_test_split iris = load_iris() X, y = iris.data, iris.target X_train, X_test, y_train, y_test = train_test_split( X, y, test_size=0.33, random_state=1) Use scikit-learn's GridSearch to tune the hyperparameters of the LogisticRegression estimator inside the ExhaustiveFeatureSelector and use it for prediction in the pipeline. Note that the clone_estimator attribute needs to be set to False . from sklearn.model_selection import GridSearchCV from sklearn.pipeline import make_pipeline from sklearn.linear_model import LogisticRegression from mlxtend.feature_selection import ExhaustiveFeatureSelector as EFS lr = LogisticRegression(multi_class='multinomial', solver='newton-cg', random_state=123) efs1 = EFS(estimator=lr, min_features=2, max_features=3, scoring='accuracy', print_progress=False, clone_estimator=False, cv=5, n_jobs=1) pipe = make_pipeline(efs1, lr) param_grid = {'exhaustivefeatureselector__estimator__C': [0.1, 1.0, 10.0]} gs = GridSearchCV(estimator=pipe, param_grid=param_grid, scoring='accuracy', n_jobs=1, cv=2, verbose=1, refit=False) # run gridearch gs = gs.fit(X_train, y_train) Fitting 2 folds for each of 3 candidates, totalling 6 fits ... and the \"best\" parameters determined by GridSearch are ... print(\"Best parameters via GridSearch\", gs.best_params_) Best parameters via GridSearch {'exhaustivefeatureselector__estimator__C': 0.1}","title":"Example 6 - Exhaustive feature selection and GridSearch"},{"location":"user_guide/feature_selection/ExhaustiveFeatureSelector/#obtaining-the-best-k-feature-indices-after-gridsearch","text":"If we are interested in the best k best feature indices via SequentialFeatureSelection.best_idx_ , we have to initialize a GridSearchCV object with refit=True . Now, the grid search object will take the complete training dataset and the best parameters, which it found via cross-validation, to train the estimator pipeline. gs = GridSearchCV(estimator=pipe, param_grid=param_grid, scoring='accuracy', n_jobs=1, cv=2, verbose=1, refit=True) After running the grid search, we can access the individual pipeline objects of the best_estimator_ via the steps attribute. gs = gs.fit(X_train, y_train) gs.best_estimator_.steps Fitting 2 folds for each of 3 candidates, totalling 6 fits [('exhaustivefeatureselector', ExhaustiveFeatureSelector(clone_estimator=False, estimator=LogisticRegression(C=0.1, multi_class='multinomial', random_state=123, solver='newton-cg'), feature_groups=[[0], [1], [2], [3]], max_features=3, min_features=2, print_progress=False)), ('logisticregression', LogisticRegression(multi_class='multinomial', random_state=123, solver='newton-cg'))] Via sub-indexing, we can then obtain the best-selected feature subset: print('Best features:', gs.best_estimator_.steps[0][1].best_idx_) Best features: (2, 3) During cross-validation, this feature combination had a CV accuracy of: print('Best score:', gs.best_score_) Best score: 0.96 gs.best_params_ {'exhaustivefeatureselector__estimator__C': 0.1} Alternatively , if we can set the \"best grid search parameters\" in our pipeline manually if we ran GridSearchCV with refit=False . It should yield the same results: pipe.set_params(**gs.best_params_).fit(X_train, y_train) print('Best features:', pipe.steps[0][1].best_idx_) Best features: (2, 3)","title":"Obtaining the best k feature indices after GridSearch"},{"location":"user_guide/feature_selection/ExhaustiveFeatureSelector/#example-7-exhaustive-feature-selection-with-loocv","text":"The ExhaustiveFeatureSelector is not restricted to k-fold cross-validation. You can use any type of cross-validation method that supports the general scikit-learn cross-validation API. The following example illustrates the use of scikit-learn's LeaveOneOut cross-validation method in combination with the exhaustive feature selector. from sklearn.neighbors import KNeighborsClassifier from sklearn.datasets import load_iris from mlxtend.feature_selection import ExhaustiveFeatureSelector as EFS from sklearn.model_selection import LeaveOneOut iris = load_iris() X = iris.data y = iris.target knn = KNeighborsClassifier(n_neighbors=3) efs1 = EFS(knn, min_features=1, max_features=4, scoring='accuracy', print_progress=True, cv=LeaveOneOut()) ### Use cross-validation generator here efs1 = efs1.fit(X, y) print('Best accuracy score: %.2f' % efs1.best_score_) print('Best subset (indices):', efs1.best_idx_) print('Best subset (corresponding names):', efs1.best_feature_names_) Features: 15/15 Best accuracy score: 0.96 Best subset (indices): (3,) Best subset (corresponding names): ('3',)","title":"Example 7 - Exhaustive Feature Selection with LOOCV"},{"location":"user_guide/feature_selection/ExhaustiveFeatureSelector/#example-8-interrupting-long-runs-for-intermediate-results","text":"If your run is taking too long, it is possible to trigger a KeyboardInterrupt (e.g., ctrl+c on a Mac, or interrupting the cell in a Jupyter notebook) to obtain temporary results. Toy dataset from sklearn.datasets import make_classification from sklearn.model_selection import train_test_split X, y = make_classification( n_samples=200000, n_features=6, n_informative=2, n_redundant=1, n_repeated=1, n_clusters_per_class=2, flip_y=0.05, class_sep=0.5, random_state=123, ) X_train, X_test, y_train, y_test = train_test_split( X, y, test_size=0.2, random_state=123 ) Long run with interruption from mlxtend.feature_selection import ExhaustiveFeatureSelector as EFS from sklearn.linear_model import LogisticRegression model = LogisticRegression(max_iter=10000) efs1 = EFS(model, min_features=1, max_features=4, print_progress=True, scoring='accuracy') efs1 = efs1.fit(X_train, y_train) Features: 56/56 Finalizing the fit Note that the feature selection run hasn't finished, so certain attributes may not be available. In order to use the EFS instance, it is recommended to call finalize_fit , which will make EFS estimator appear as \"fitted\" process the temporary results: efs1.finalize_fit() print('Best accuracy score: %.2f' % efs1.best_score_) print('Best subset (indices):', efs1.best_idx_) Best accuracy score: 0.73 Best subset (indices): (1, 2)","title":"Example 8 - Interrupting Long Runs for Intermediate Results"},{"location":"user_guide/feature_selection/ExhaustiveFeatureSelector/#example-9-working-with-feature-groups","text":"Since mlxtend v0.21.0, it is possible to specify feature groups. Feature groups allow you to group certain features together, such that they are always selected as a group. This can be very useful in contexts similar to one-hot encoding -- if you want to treat the one-hot encoded feature as a single feature: In the following example, we specify sepal length and sepal width as a feature group so that they are always selected together: from sklearn.datasets import load_iris import pandas as pd iris = load_iris() X = iris.data y = iris.target X_df = pd.DataFrame(X, columns=['sepal len', 'petal len', 'sepal wid', 'petal wid']) X_df.head() .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } sepal len petal len sepal wid petal wid 0 5.1 3.5 1.4 0.2 1 4.9 3.0 1.4 0.2 2 4.7 3.2 1.3 0.2 3 4.6 3.1 1.5 0.2 4 5.0 3.6 1.4 0.2 from sklearn.neighbors import KNeighborsClassifier from mlxtend.feature_selection import ExhaustiveFeatureSelector as EFS knn = KNeighborsClassifier(n_neighbors=3) efs1 = EFS(knn, min_features=2, max_features=2, scoring='accuracy', feature_groups=[['sepal len', 'sepal wid'], ['petal len'], ['petal wid']], cv=3) efs1 = efs1.fit(X_df, y) print('Best accuracy score: %.2f' % efs1.best_score_) print('Best subset (indices):', efs1.best_idx_) print('Best subset (corresponding names):', efs1.best_feature_names_) Features: 3/3 Best accuracy score: 0.97 Best subset (indices): (0, 2, 3) Best subset (corresponding names): ('sepal len', 'sepal wid', 'petal wid') Notice that the returned number of features is 3, since the number of min_features and max_features corresponds to the number of feature groups. I.e., we have 2 feature groups in ['sepal len', 'sepal wid'], ['petal wid'] , but it expands to 3 features.","title":"Example 9 - Working with Feature Groups"},{"location":"user_guide/feature_selection/ExhaustiveFeatureSelector/#api","text":"ExhaustiveFeatureSelector(estimator, min_features=1, max_features=1, print_progress=True, scoring='accuracy', cv=5, n_jobs=1, pre_dispatch='2 n_jobs', clone_estimator=True, fixed_features=None, feature_groups=None)* Exhaustive Feature Selection for Classification and Regression. (new in v0.4.3) Parameters estimator : scikit-learn classifier or regressor min_features : int (default: 1) Minumum number of features to select max_features : int (default: 1) Maximum number of features to select. If parameter feature_groups is not None, the number of features is equal to the number of feature groups, i.e. len(feature_groups) . For example, if feature_groups = [[0], [1], [2, 3], [4]] , then the max_features value cannot exceed 4. print_progress : bool (default: True) Prints progress as the number of epochs to stderr. scoring : str, (default='accuracy') Scoring metric in {accuracy, f1, precision, recall, roc_auc} for classifiers, {'mean_absolute_error', 'mean_squared_error', 'median_absolute_error', 'r2'} for regressors, or a callable object or function with signature scorer(estimator, X, y) . cv : int (default: 5) Scikit-learn cross-validation generator or int . If estimator is a classifier (or y consists of integer class labels), stratified k-fold is performed, and regular k-fold cross-validation otherwise. No cross-validation if cv is None, False, or 0. n_jobs : int (default: 1) The number of CPUs to use for evaluating different feature subsets in parallel. -1 means 'all CPUs'. pre_dispatch : int, or string (default: '2*n_jobs') Controls the number of jobs that get dispatched during parallel execution if n_jobs > 1 or n_jobs=-1 . Reducing this number can be useful to avoid an explosion of memory consumption when more jobs get dispatched than CPUs can process. This parameter can be: None, in which case all the jobs are immediately created and spawned. Use this for lightweight and fast-running jobs, to avoid delays due to on-demand spawning of the jobs An int, giving the exact number of total jobs that are spawned A string, giving an expression as a function of n_jobs, as in 2*n_jobs clone_estimator : bool (default: True) Clones estimator if True; works with the original estimator instance if False. Set to False if the estimator doesn't implement scikit-learn's set_params and get_params methods. In addition, it is required to set cv=0, and n_jobs=1. fixed_features : tuple (default: None) If not None , the feature indices provided as a tuple will be regarded as fixed by the feature selector. For example, if fixed_features=(1, 3, 7) , the 2nd, 4th, and 8th feature are guaranteed to be present in the solution. Note that if fixed_features is not None , make sure that the number of features to be selected is greater than len(fixed_features) . In other words, ensure that k_features > len(fixed_features) . feature_groups : list or None (default: None) Optional argument for treating certain features as a group. This means, the features within a group are always selected together, never split. For example, feature_groups=[[1], [2], [3, 4, 5]] specifies 3 feature groups.In this case, possible feature selection results with k_features=2 are [[1], [2] , [[1], [3, 4, 5]] , or [[2], [3, 4, 5]] . Feature groups can be useful for interpretability, for example, if features 3, 4, 5 are one-hot encoded features. (For more details, please read the notes at the bottom of this docstring). New in mlxtend v. 0.21.0. Attributes best_idx_ : array-like, shape = [n_predictions] Feature Indices of the selected feature subsets. best_feature_names_ : array-like, shape = [n_predictions] Feature names of the selected feature subsets. If pandas DataFrames are used in the fit method, the feature names correspond to the column names. Otherwise, the feature names are string representation of the feature array indices. New in v 0.13.0. best_score_ : float Cross validation average score of the selected subset. subsets_ : dict A dictionary of selected feature subsets during the exhaustive selection, where the dictionary keys are the lengths k of these feature subsets. The dictionary values are dictionaries themselves with the following keys: 'feature_idx' (tuple of indices of the feature subset) 'feature_names' (tuple of feature names of the feat. subset) 'cv_scores' (list individual cross-validation scores) 'avg_score' (average cross-validation score) Note that if pandas DataFrames are used in the fit method, the 'feature_names' correspond to the column names. Otherwise, the feature names are string representation of the feature array indices. The 'feature_names' is new in v. 0.13.0. Notes (1) If parameter feature_groups is not None, the number of features is equal to the number of feature groups, i.e. len(feature_groups) . For example, if feature_groups = [[0], [1], [2, 3], [4]] , then the max_features value cannot exceed 4. (2) Although two or more individual features may be considered as one group throughout the feature-selection process, it does not mean the individual features of that group have the same impact on the outcome. For instance, in linear regression, the coefficient of the feature 2 and 3 can be different even if they are considered as one group in feature_groups. (3) If both fixed_features and feature_groups are specified, ensure that each feature group contains the fixed_features selection. E.g., for a 3-feature set fixed_features=[0, 1] and feature_groups=[[0, 1], [2]] is valid; fixed_features=[0, 1] and feature_groups=[[0], [1, 2]] is not valid. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/feature_selection/ExhaustiveFeatureSelector/","title":"API"},{"location":"user_guide/feature_selection/ExhaustiveFeatureSelector/#methods","text":"fit(X, y, groups=None, fit_params) Perform feature selection and learn model from training data. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. New in v 0.13.0: pandas DataFrames are now also accepted as argument for X. y : array-like, shape = [n_samples] Target values. groups : array-like, with shape (n_samples,), optional Group labels for the samples used while splitting the dataset into train/test set. Passed to the fit method of the cross-validator. fit_params : dict of string -> object, optional Parameters to pass to to the fit method of classifier. Returns self : object fit_transform(X, y, groups=None, fit_params) Fit to training data and return the best selected features from X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. New in v 0.13.0: pandas DataFrames are now also accepted as argument for X. y : array-like, shape = [n_samples] Target values. groups : array-like, with shape (n_samples,), optional Group labels for the samples used while splitting the dataset into train/test set. Passed to the fit method of the cross-validator. fit_params : dict of string -> object, optional Parameters to pass to to the fit method of classifier. Returns Feature subset of X, shape={n_samples, k_features} get_metric_dict(confidence_interval=0.95) Return metric dictionary Parameters confidence_interval : float (default: 0.95) A positive float between 0.0 and 1.0 to compute the confidence interval bounds of the CV score averages. Returns Dictionary with items where each dictionary value is a list with the number of iterations (number of feature subsets) as its length. The dictionary keys corresponding to these lists are as follows: 'feature_idx': tuple of the indices of the feature subset 'cv_scores': list with individual CV scores 'avg_score': of CV average scores 'std_dev': standard deviation of the CV score average 'std_err': standard error of the CV score average 'ci_bound': confidence interval bound of the CV score average get_params(deep=True) Get parameters for this estimator. Parameters deep : bool, default=True If True, will return the parameters for this estimator and contained subobjects that are estimators. Returns params : dict Parameter names mapped to their values. set_params( params) Set the parameters of this estimator. The method works on simple estimators as well as on nested objects (such as :class:`~sklearn.pipeline.Pipeline`). The latter have parameters of the form ``<component>__<parameter>`` so that it's possible to update each component of a nested object. Parameters **params : dict Estimator parameters. Returns self : estimator instance Estimator instance. transform(X) Return the best selected features from X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. New in v 0.13.0: pandas DataFrames are now also accepted as argument for X. Returns Feature subset of X, shape={n_samples, k_features}","title":"Methods"},{"location":"user_guide/feature_selection/SequentialFeatureSelector/","text":"SequentialFeatureSelector: The popular forward and backward feature selection approaches (including floating variants) Implementation of sequential feature algorithms (SFAs) -- greedy search algorithms -- that have been developed as a suboptimal solution to the computationally often not feasible exhaustive search. from mlxtend.feature_selection import SequentialFeatureSelector Overview Sequential feature selection algorithms are a family of greedy search algorithms that are used to reduce an initial d -dimensional feature space to a k -dimensional feature subspace where k < d . The motivation behind feature selection algorithms is to automatically select a subset of features most relevant to the problem. The goal of feature selection is two-fold: We want to improve the computational efficiency and reduce the model's generalization error by removing irrelevant features or noise. In addition, a wrapper approach such as sequential feature selection is advantageous if embedded feature selection -- for example, a regularization penalty like LASSO -- is not applicable. In a nutshell, SFAs remove or add one feature at a time based on the classifier performance until a feature subset of the desired size k is reached. There are four different flavors of SFAs available via the SequentialFeatureSelector : Sequential Forward Selection (SFS) Sequential Backward Selection (SBS) Sequential Forward Floating Selection (SFFS) Sequential Backward Floating Selection (SBFS) The floating variants, SFFS and SBFS, can be considered extensions to the simpler SFS and SBS algorithms. The floating algorithms have an additional exclusion or inclusion step to remove features once they were included (or excluded) so that a larger number of feature subset combinations can be sampled. It is important to emphasize that this step is conditional and only occurs if the resulting feature subset is assessed as \"better\" by the criterion function after the removal (or addition) of a particular feature. Furthermore, I added an optional check to skip the conditional exclusion steps if the algorithm gets stuck in cycles. How is this different from Recursive Feature Elimination (RFE) -- e.g., as implemented in sklearn.feature_selection.RFE ? RFE is computationally less complex using the feature weight coefficients (e.g., linear models) or feature importance (tree-based algorithms) to eliminate features recursively, whereas SFSs eliminate (or add) features based on a user-defined classifier/regression performance metric. Tutorial Videos Visual Illustration A visual illustration of the sequential backward selection process is provided below, from the paper Joe Bemister-Buffington, Alex J. Wolf, Sebastian Raschka, and Leslie A. Kuhn (2020) Machine Learning to Identify Flexibility Signatures of Class A GPCR Inhibition Biomolecules 2020, 10, 454. https://www.mdpi.com/2218-273X/10/3/454# Algorithmic Details Sequential Forward Selection (SFS) Input: Y = \\{y_1, y_2, ..., y_d\\} The SFS algorithm takes the whole d -dimensional feature set as input. Output: X_k = \\{x_j \\; | \\;j = 1, 2, ..., k; \\; x_j \\in Y\\} , where k = (0, 1, 2, ..., d) SFS returns a subset of features; the number of selected features k , where k < d , has to be specified a priori . Initialization: X_0 = \\emptyset , k = 0 We initialize the algorithm with an empty set \\emptyset (\"null set\") so that k = 0 (where k is the size of the subset). Step 1 (Inclusion): x^+ = \\text{ arg max } J(X_k + x), \\text{ where } x \\in Y - X_k X_{k+1} = X_k + x^+ k = k + 1 Go to Step 1 in this step, we add an additional feature, x^+ , to our feature subset X_k . x^+ is the feature that maximizes our criterion function, that is, the feature that is associated with the best classifier performance if it is added to X_k . We repeat this procedure until the termination criterion is satisfied. Termination: k = p We add features from the feature subset X_k until the feature subset of size k contains the number of desired features p that we specified a priori . Sequential Backward Selection (SBS) Input: the set of all features, Y = \\{y_1, y_2, ..., y_d\\} The SBS algorithm takes the whole feature set as input. Output: X_k = \\{x_j \\; | \\;j = 1, 2, ..., k; \\; x_j \\in Y\\} , where k = (0, 1, 2, ..., d) SBS returns a subset of features; the number of selected features k , where k < d , has to be specified a priori . Initialization: X_0 = Y , k = d We initialize the algorithm with the given feature set so that the k = d . Step 1 (Exclusion): x^- = \\text{ arg max } J(X_k - x), \\text{ where } x \\in X_k X_{k-1} = X_k - x^- k = k - 1 Go to Step 1 In this step, we remove a feature, x^- from our feature subset X_k . x^- is the feature that maximizes our criterion function upon re,oval, that is, the feature that is associated with the best classifier performance if it is removed from X_k . We repeat this procedure until the termination criterion is satisfied. Termination: k = p We add features from the feature subset X_k until the feature subset of size k contains the number of desired features p that we specified a priori . Sequential Backward Floating Selection (SBFS) Input: the set of all features, Y = \\{y_1, y_2, ..., y_d\\} The SBFS algorithm takes the whole feature set as input. Output: X_k = \\{x_j \\; | \\;j = 1, 2, ..., k; \\; x_j \\in Y\\} , where k = (0, 1, 2, ..., d) SBFS returns a subset of features; the number of selected features k , where k < d , has to be specified a priori . Initialization: X_0 = Y , k = d We initialize the algorithm with the given feature set so that the k = d . Step 1 (Exclusion): x^- = \\text{ arg max } J(X_k - x), \\text{ where } x \\in X_k X_{k-1} = X_k - x^- k = k - 1 Go to Step 2 In this step, we remove a feature, x^- from our feature subset X_k . x^- is the feature that maximizes our criterion function upon removal, that is, the feature that is associated with the best classifier performance if it is removed from X_k . Step 2 (Conditional Inclusion): x^+ = \\text{ arg max } J(X_k + x), \\text{ where } x \\in Y - X_k if J(X_k + x) > J(X_k) : X_{k+1} = X_k + x^+ k = k + 1 Go to Step 1 In Step 2, we search for features that improve the classifier performance if they are added back to the feature subset. If such features exist, we add the feature x^+ for which the performance improvement is maximized. If k = 2 or an improvement cannot be made (i.e., such feature x^+ cannot be found), go back to step 1; else, repeat this step. Termination: k = p We add features from the feature subset X_k until the feature subset of size k contains the number of desired features p that we specified a priori . Sequential Forward Floating Selection (SFFS) Input: the set of all features, Y = \\{y_1, y_2, ..., y_d\\} The SFFS algorithm takes the whole feature set as input, if our feature space consists of, e.g. 10, if our feature space consists of 10 dimensions ( d = 10 ). Output: a subset of features, X_k = \\{x_j \\; | \\;j = 1, 2, ..., k; \\; x_j \\in Y\\} , where k = (0, 1, 2, ..., d) The returned output of the algorithm is a subset of the feature space of a specified size. E.g., a subset of 5 features from a 10-dimensional feature space ( k = 5, d = 10 ). Initialization: X_0 = \\emptyset , k = 0 We initialize the algorithm with an empty set (\"null set\") so that the k = 0 (where k is the size of the subset) Step 1 (Inclusion): x^+ = \\text{ arg max } J(X_k + x), \\text{ where } x \\in Y - X_k X_{k+1} = X_k + x^+ k = k + 1 Go to Step 2 Step 2 (Conditional Exclusion): x^- = \\text{ arg max } J(X_k - x), \\text{ where } x \\in X_k if \\; J(X_k - x) > J(X_k) : X_{k-1} = X_k - x^- k = k - 1 Go to Step 1 In step 1, we include the feature from the feature space that leads to the best performance increase for our feature subset (assessed by the criterion function ). Then, we go over to step 2 In step 2, we only remove a feature if the resulting subset would gain an increase in performance. If k = 2 or an improvement cannot be made (i.e., such feature x^+ cannot be found), go back to step 1; else, repeat this step. Steps 1 and 2 are repeated until the Termination criterion is reached. Termination: stop when k equals the number of desired features References Ferri, F. J., Pudil P., Hatef, M., Kittler, J. (1994). \"Comparative study of techniques for large-scale feature selection.\" Pattern Recognition in Practice IV : 403-413. Pudil, P., Novovi\u010dov\u00e1, J., & Kittler, J. (1994). \"Floating search methods in feature selection.\" Pattern recognition letters 15.11 (1994): 1119-1125. Example 1 - A simple Sequential Forward Selection example Initializing a simple classifier from scikit-learn: from sklearn.neighbors import KNeighborsClassifier from sklearn.datasets import load_iris iris = load_iris() X = iris.data y = iris.target knn = KNeighborsClassifier(n_neighbors=4) We start by selection the \"best\" 3 features from the Iris dataset via Sequential Forward Selection (SFS). Here, we set forward=True and floating=False . By choosing cv=0 , we don't perform any cross-validation, therefore, the performance (here: 'accuracy' ) is computed entirely on the training set. from mlxtend.feature_selection import SequentialFeatureSelector as SFS sfs1 = SFS(knn, k_features=3, forward=True, floating=False, verbose=2, scoring='accuracy', cv=0) sfs1 = sfs1.fit(X, y) [Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers. [Parallel(n_jobs=1)]: Done 1 out of 1 | elapsed: 0.0s remaining: 0.0s [Parallel(n_jobs=1)]: Done 4 out of 4 | elapsed: 0.0s finished [2023-05-17 08:36:17] Features: 1/3 -- score: 0.96[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers. [Parallel(n_jobs=1)]: Done 1 out of 1 | elapsed: 0.0s remaining: 0.0s [Parallel(n_jobs=1)]: Done 3 out of 3 | elapsed: 0.0s finished [2023-05-17 08:36:17] Features: 2/3 -- score: 0.9733333333333334[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers. [Parallel(n_jobs=1)]: Done 1 out of 1 | elapsed: 0.0s remaining: 0.0s [Parallel(n_jobs=1)]: Done 2 out of 2 | elapsed: 0.0s finished [2023-05-17 08:36:17] Features: 3/3 -- score: 0.9733333333333334 Via the subsets_ attribute, we can take a look at the selected feature indices at each step: sfs1.subsets_ {1: {'feature_idx': (3,), 'cv_scores': array([0.96]), 'avg_score': 0.96, 'feature_names': ('3',)}, 2: {'feature_idx': (2, 3), 'cv_scores': array([0.97333333]), 'avg_score': 0.9733333333333334, 'feature_names': ('2', '3')}, 3: {'feature_idx': (1, 2, 3), 'cv_scores': array([0.97333333]), 'avg_score': 0.9733333333333334, 'feature_names': ('1', '2', '3')}} sfs1 = sfs1.fit(X, y) sfs1.subsets_ [Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers. [Parallel(n_jobs=1)]: Done 1 out of 1 | elapsed: 0.0s remaining: 0.0s [Parallel(n_jobs=1)]: Done 4 out of 4 | elapsed: 0.0s finished [2023-05-17 08:36:17] Features: 1/3 -- score: 0.96[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers. [Parallel(n_jobs=1)]: Done 1 out of 1 | elapsed: 0.0s remaining: 0.0s [Parallel(n_jobs=1)]: Done 3 out of 3 | elapsed: 0.0s finished [2023-05-17 08:36:17] Features: 2/3 -- score: 0.9733333333333334[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers. [Parallel(n_jobs=1)]: Done 1 out of 1 | elapsed: 0.0s remaining: 0.0s [Parallel(n_jobs=1)]: Done 2 out of 2 | elapsed: 0.0s finished [2023-05-17 08:36:17] Features: 3/3 -- score: 0.9733333333333334 {1: {'feature_idx': (3,), 'cv_scores': array([0.96]), 'avg_score': 0.96, 'feature_names': ('3',)}, 2: {'feature_idx': (2, 3), 'cv_scores': array([0.97333333]), 'avg_score': 0.9733333333333334, 'feature_names': ('2', '3')}, 3: {'feature_idx': (1, 2, 3), 'cv_scores': array([0.97333333]), 'avg_score': 0.9733333333333334, 'feature_names': ('1', '2', '3')}} Furthermore, we can access the indices of the 3 best features directly via the k_feature_idx_ attribute: sfs1.k_feature_idx_ (1, 2, 3) Finally, the prediction score for these 3 features can be accesses via k_score_ : sfs1.k_score_ 0.9733333333333334 Feature Names When working with large datasets, the feature indices might be hard to interpret. In this case, we recommend using pandas DataFrames with distinct column names as input: import pandas as pd df_X = pd.DataFrame(X, columns=[\"Sepal length\", \"Sepal width\", \"Petal length\", \"Petal width\"]) df_X.head() .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } Sepal length Sepal width Petal length Petal width 0 5.1 3.5 1.4 0.2 1 4.9 3.0 1.4 0.2 2 4.7 3.2 1.3 0.2 3 4.6 3.1 1.5 0.2 4 5.0 3.6 1.4 0.2 sfs1 = sfs1.fit(df_X, y) print('Best accuracy score: %.2f' % sfs1.k_score_) print('Best subset (indices):', sfs1.k_feature_idx_) print('Best subset (corresponding names):', sfs1.k_feature_names_) [Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers. Best accuracy score: 0.97 Best subset (indices): (1, 2, 3) Best subset (corresponding names): ('Sepal width', 'Petal length', 'Petal width') [Parallel(n_jobs=1)]: Done 1 out of 1 | elapsed: 0.0s remaining: 0.0s [Parallel(n_jobs=1)]: Done 4 out of 4 | elapsed: 0.0s finished [2023-05-17 08:36:17] Features: 1/3 -- score: 0.96[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers. [Parallel(n_jobs=1)]: Done 1 out of 1 | elapsed: 0.0s remaining: 0.0s [Parallel(n_jobs=1)]: Done 3 out of 3 | elapsed: 0.0s finished [2023-05-17 08:36:17] Features: 2/3 -- score: 0.9733333333333334[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers. [Parallel(n_jobs=1)]: Done 1 out of 1 | elapsed: 0.0s remaining: 0.0s [Parallel(n_jobs=1)]: Done 2 out of 2 | elapsed: 0.0s finished [2023-05-17 08:36:17] Features: 3/3 -- score: 0.9733333333333334 Example 2 - Toggling between SFS, SBS, SFFS, and SBFS Using the forward and floating parameters, we can toggle between SFS, SBS, SFFS, and SBFS as shown below. Note that we are performing (stratified) 4-fold cross-validation for more robust estimates in contrast to Example 1. Via n_jobs=-1 , we choose to run the cross-validation on all our available CPU cores. # Sequential Forward Selection sfs = SFS(knn, k_features=3, forward=True, floating=False, scoring='accuracy', cv=4, n_jobs=-1) sfs = sfs.fit(X, y) print('\\nSequential Forward Selection (k=3):') print(sfs.k_feature_idx_) print('CV Score:') print(sfs.k_score_) ################################################### # Sequential Backward Selection sbs = SFS(knn, k_features=3, forward=False, floating=False, scoring='accuracy', cv=4, n_jobs=-1) sbs = sbs.fit(X, y) print('\\nSequential Backward Selection (k=3):') print(sbs.k_feature_idx_) print('CV Score:') print(sbs.k_score_) ################################################### # Sequential Forward Floating Selection sffs = SFS(knn, k_features=3, forward=True, floating=True, scoring='accuracy', cv=4, n_jobs=-1) sffs = sffs.fit(X, y) print('\\nSequential Forward Floating Selection (k=3):') print(sffs.k_feature_idx_) print('CV Score:') print(sffs.k_score_) ################################################### # Sequential Backward Floating Selection sbfs = SFS(knn, k_features=3, forward=False, floating=True, scoring='accuracy', cv=4, n_jobs=-1) sbfs = sbfs.fit(X, y) print('\\nSequential Backward Floating Selection (k=3):') print(sbfs.k_feature_idx_) print('CV Score:') print(sbfs.k_score_) Sequential Forward Selection (k=3): (1, 2, 3) CV Score: 0.9731507823613088 Sequential Backward Selection (k=3): (1, 2, 3) CV Score: 0.9731507823613088 Sequential Forward Floating Selection (k=3): (1, 2, 3) CV Score: 0.9731507823613088 Sequential Backward Floating Selection (k=3): (1, 2, 3) CV Score: 0.9731507823613088 In this simple scenario, selecting the best 3 features out of the 4 available features in the Iris set, we end up with similar results regardless of which sequential selection algorithms we used. Example 3 - Visualizing the results in DataFrames For our convenience, we can visualize the output from the feature selection in a pandas DataFrame format using the get_metric_dict method of the SequentialFeatureSelector object. The columns std_dev and std_err represent the standard deviation and standard errors of the cross-validation scores, respectively. Below, we see the DataFrame of the Sequential Forward Selector from Example 2: import pandas as pd pd.DataFrame.from_dict(sfs.get_metric_dict()).T .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } feature_idx cv_scores avg_score feature_names ci_bound std_dev std_err 1 (3,) [0.9736842105263158, 0.9473684210526315, 0.918... 0.959993 (3,) 0.048319 0.030143 0.017403 2 (2, 3) [0.9736842105263158, 0.9473684210526315, 0.918... 0.959993 (2, 3) 0.048319 0.030143 0.017403 3 (1, 2, 3) [0.9736842105263158, 1.0, 0.9459459459459459, ... 0.973151 (1, 2, 3) 0.030639 0.019113 0.011035 Now, let's compare it to the Sequential Backward Selector: pd.DataFrame.from_dict(sbs.get_metric_dict()).T .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } feature_idx cv_scores avg_score feature_names ci_bound std_dev std_err 4 (0, 1, 2, 3) [0.9736842105263158, 0.9473684210526315, 0.918... 0.953236 (0, 1, 2, 3) 0.03602 0.022471 0.012974 3 (1, 2, 3) [0.9736842105263158, 1.0, 0.9459459459459459, ... 0.973151 (1, 2, 3) 0.030639 0.019113 0.011035 We can see that both SFS and SBFS found the same \"best\" 3 features, however, the intermediate steps where obviously different. The ci_bound column in the DataFrames above represents the confidence interval around the computed cross-validation scores. By default, a confidence interval of 95% is used, but we can use different confidence bounds via the confidence_interval parameter. E.g., the confidence bounds for a 90% confidence interval can be obtained as follows: pd.DataFrame.from_dict(sbs.get_metric_dict(confidence_interval=0.90)).T .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } feature_idx cv_scores avg_score feature_names ci_bound std_dev std_err 4 (0, 1, 2, 3) [0.9736842105263158, 0.9473684210526315, 0.918... 0.953236 (0, 1, 2, 3) 0.027658 0.022471 0.012974 3 (1, 2, 3) [0.9736842105263158, 1.0, 0.9459459459459459, ... 0.973151 (1, 2, 3) 0.023525 0.019113 0.011035 Example 4 - Plotting the results After importing the little helper function plotting.plot_sequential_feature_selection , we can also visualize the results using matplotlib figures. from mlxtend.plotting import plot_sequential_feature_selection as plot_sfs import matplotlib.pyplot as plt sfs = SFS(knn, k_features=4, forward=True, floating=False, scoring='accuracy', verbose=2, cv=5) sfs = sfs.fit(X, y) fig1 = plot_sfs(sfs.get_metric_dict(), kind='std_dev') plt.ylim([0.8, 1]) plt.title('Sequential Forward Selection (w. StdDev)') plt.grid() plt.show() [Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers. [Parallel(n_jobs=1)]: Done 1 out of 1 | elapsed: 0.0s remaining: 0.0s [Parallel(n_jobs=1)]: Done 4 out of 4 | elapsed: 0.0s finished [2023-05-17 08:36:18] Features: 1/4 -- score: 0.96[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers. [Parallel(n_jobs=1)]: Done 1 out of 1 | elapsed: 0.0s remaining: 0.0s [Parallel(n_jobs=1)]: Done 3 out of 3 | elapsed: 0.0s finished [2023-05-17 08:36:18] Features: 2/4 -- score: 0.9666666666666668[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers. [Parallel(n_jobs=1)]: Done 1 out of 1 | elapsed: 0.0s remaining: 0.0s [Parallel(n_jobs=1)]: Done 2 out of 2 | elapsed: 0.0s finished [2023-05-17 08:36:18] Features: 3/4 -- score: 0.9533333333333334[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers. [Parallel(n_jobs=1)]: Done 1 out of 1 | elapsed: 0.0s remaining: 0.0s [Parallel(n_jobs=1)]: Done 1 out of 1 | elapsed: 0.0s finished [2023-05-17 08:36:18] Features: 4/4 -- score: 0.9733333333333334 Example 5 - Sequential Feature Selection for Regression Similar to the classification examples above, the SequentialFeatureSelector also supports scikit-learn's estimators for regression. from sklearn.linear_model import LinearRegression from sklearn.datasets import fetch_california_housing data = fetch_california_housing() X, y = data.data, data.target lr = LinearRegression() sfs = SFS(lr, k_features=8, forward=True, floating=False, scoring='neg_mean_squared_error', cv=10) sfs = sfs.fit(X, y) fig = plot_sfs(sfs.get_metric_dict(), kind='std_err') plt.title('Sequential Forward Selection (w. StdErr)') plt.grid() plt.show() Example 6 -- Feature Selection with Fixed Train/Validation Splits If you do not wish to use cross-validation (here: k-fold cross-validation, i.e., rotating training and validation folds), you can use the PredefinedHoldoutSplit class to specify your own, fixed training and validation split. from sklearn.datasets import load_iris from mlxtend.evaluate import PredefinedHoldoutSplit import numpy as np iris = load_iris() X = iris.data y = iris.target rng = np.random.RandomState(123) my_validation_indices = rng.permutation(np.arange(150))[:30] print(my_validation_indices) [ 72 112 132 88 37 138 87 42 8 90 141 33 59 116 135 104 36 13 63 45 28 133 24 127 46 20 31 121 117 4] from sklearn.neighbors import KNeighborsClassifier from mlxtend.feature_selection import SequentialFeatureSelector as SFS knn = KNeighborsClassifier(n_neighbors=4) piter = PredefinedHoldoutSplit(my_validation_indices) sfs1 = SFS(knn, k_features=3, forward=True, floating=False, verbose=2, scoring='accuracy', cv=piter) sfs1 = sfs1.fit(X, y) [Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers. [Parallel(n_jobs=1)]: Done 1 out of 1 | elapsed: 0.0s remaining: 0.0s [Parallel(n_jobs=1)]: Done 4 out of 4 | elapsed: 0.0s finished [2023-05-17 08:36:19] Features: 1/3 -- score: 0.9666666666666667[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers. [Parallel(n_jobs=1)]: Done 1 out of 1 | elapsed: 0.0s remaining: 0.0s [Parallel(n_jobs=1)]: Done 3 out of 3 | elapsed: 0.0s finished [2023-05-17 08:36:19] Features: 2/3 -- score: 0.9666666666666667[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers. [Parallel(n_jobs=1)]: Done 1 out of 1 | elapsed: 0.0s remaining: 0.0s [Parallel(n_jobs=1)]: Done 2 out of 2 | elapsed: 0.0s finished [2023-05-17 08:36:19] Features: 3/3 -- score: 0.9666666666666667 Example 7 -- Using the Selected Feature Subset For Making New Predictions # Initialize the dataset from sklearn.neighbors import KNeighborsClassifier from sklearn.datasets import load_iris from sklearn.model_selection import train_test_split iris = load_iris() X, y = iris.data, iris.target X_train, X_test, y_train, y_test = train_test_split( X, y, test_size=0.33, random_state=1) knn = KNeighborsClassifier(n_neighbors=4) # Select the \"best\" three features via # 5-fold cross-validation on the training set. from mlxtend.feature_selection import SequentialFeatureSelector as SFS sfs1 = SFS(knn, k_features=3, forward=True, floating=False, scoring='accuracy', cv=5) sfs1 = sfs1.fit(X_train, y_train) print('Selected features:', sfs1.k_feature_idx_) Selected features: (1, 2, 3) # Generate the new subsets based on the selected features # Note that the transform call is equivalent to # X_train[:, sfs1.k_feature_idx_] X_train_sfs = sfs1.transform(X_train) X_test_sfs = sfs1.transform(X_test) # Fit the estimator using the new feature subset # and make a prediction on the test data knn.fit(X_train_sfs, y_train) y_pred = knn.predict(X_test_sfs) # Compute the accuracy of the prediction acc = float((y_test == y_pred).sum()) / y_pred.shape[0] print('Test set accuracy: %.2f %%' % (acc * 100)) Test set accuracy: 96.00 % Example 8 -- Sequential Feature Selection and GridSearch In the following example, we are tuning the SFS's estimator using GridSearch. To avoid unwanted behavior or side-effects, it's recommended to use the estimator inside and outside of SFS as separate instances. # Initialize the dataset from sklearn.neighbors import KNeighborsClassifier from sklearn.datasets import load_iris from sklearn.model_selection import train_test_split iris = load_iris() X, y = iris.data, iris.target X_train, X_test, y_train, y_test = train_test_split( X, y, test_size=0.2, random_state=123) from sklearn.model_selection import GridSearchCV from sklearn.pipeline import Pipeline from mlxtend.feature_selection import SequentialFeatureSelector as SFS import mlxtend knn1 = KNeighborsClassifier() knn2 = KNeighborsClassifier() sfs1 = SFS(estimator=knn1, k_features=3, forward=True, floating=False, scoring='accuracy', cv=5) pipe = Pipeline([('sfs', sfs1), ('knn2', knn2)]) param_grid = { 'sfs__k_features': [1, 2, 3], 'sfs__estimator__n_neighbors': [3, 4, 7], # inner knn 'knn2__n_neighbors': [3, 4, 7] # outer knn } gs = GridSearchCV(estimator=pipe, param_grid=param_grid, scoring='accuracy', n_jobs=1, cv=5, refit=False) # run gridearch gs = gs.fit(X_train, y_train) Let's take a look at the suggested hyperparameters below: for i in range(len(gs.cv_results_['params'])): print(gs.cv_results_['params'][i], 'test acc.:', gs.cv_results_['mean_test_score'][i]) The \"best\" parameters determined by GridSearch are ... print(\"Best parameters via GridSearch\", gs.best_params_) Best parameters via GridSearch {'knn2__n_neighbors': 7, 'sfs__estimator__n_neighbors': 3, 'sfs__k_features': 3} pipe.set_params(**gs.best_params_).fit(X_train, y_train) #sk-container-id-1 {color: black;background-color: white;}#sk-container-id-1 pre{padding: 0;}#sk-container-id-1 div.sk-toggleable {background-color: white;}#sk-container-id-1 label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.3em;box-sizing: border-box;text-align: center;}#sk-container-id-1 label.sk-toggleable__label-arrow:before {content: \"\u25b8\";float: left;margin-right: 0.25em;color: #696969;}#sk-container-id-1 label.sk-toggleable__label-arrow:hover:before {color: black;}#sk-container-id-1 div.sk-estimator:hover label.sk-toggleable__label-arrow:before {color: black;}#sk-container-id-1 div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}#sk-container-id-1 div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}#sk-container-id-1 input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}#sk-container-id-1 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {content: \"\u25be\";}#sk-container-id-1 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}#sk-container-id-1 div.sk-estimator {font-family: monospace;background-color: #f0f8ff;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;margin-bottom: 0.5em;}#sk-container-id-1 div.sk-estimator:hover {background-color: #d4ebff;}#sk-container-id-1 div.sk-parallel-item::after {content: \"\";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}#sk-container-id-1 div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 div.sk-serial::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: 0;}#sk-container-id-1 div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;padding-right: 0.2em;padding-left: 0.2em;position: relative;}#sk-container-id-1 div.sk-item {position: relative;z-index: 1;}#sk-container-id-1 div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;position: relative;}#sk-container-id-1 div.sk-item::before, #sk-container-id-1 div.sk-parallel-item::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: -1;}#sk-container-id-1 div.sk-parallel-item {display: flex;flex-direction: column;z-index: 1;position: relative;background-color: white;}#sk-container-id-1 div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}#sk-container-id-1 div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}#sk-container-id-1 div.sk-parallel-item:only-child::after {width: 0;}#sk-container-id-1 div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0 0.4em 0.5em 0.4em;box-sizing: border-box;padding-bottom: 0.4em;background-color: white;}#sk-container-id-1 div.sk-label label {font-family: monospace;font-weight: bold;display: inline-block;line-height: 1.2em;}#sk-container-id-1 div.sk-label-container {text-align: center;}#sk-container-id-1 div.sk-container {/* jupyter's `normalize.less` sets `[hidden] { display: none; }` but bootstrap.min.css set `[hidden] { display: none !important; }` so we also need the `!important` here to be able to override the default hidden behavior on the sphinx rendered scikit-learn.org. See: https://github.com/scikit-learn/scikit-learn/issues/21755 */display: inline-block !important;position: relative;}#sk-container-id-1 div.sk-text-repr-fallback {display: none;} Pipeline(steps=[('sfs', SequentialFeatureSelector(estimator=KNeighborsClassifier(n_neighbors=3), k_features=(3, 3), scoring='accuracy')), ('knn2', KNeighborsClassifier(n_neighbors=7))]) In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org. Pipeline Pipeline(steps=[('sfs', SequentialFeatureSelector(estimator=KNeighborsClassifier(n_neighbors=3), k_features=(3, 3), scoring='accuracy')), ('knn2', KNeighborsClassifier(n_neighbors=7))]) sfs: SequentialFeatureSelector SequentialFeatureSelector(estimator=KNeighborsClassifier(n_neighbors=3), k_features=(3, 3), scoring='accuracy') estimator: KNeighborsClassifier KNeighborsClassifier(n_neighbors=3) KNeighborsClassifier KNeighborsClassifier(n_neighbors=3) KNeighborsClassifier KNeighborsClassifier(n_neighbors=7) Example 9 -- Selecting the \"best\" feature combination in a k-range If k_features is set to to a tuple (min_k, max_k) (new in 0.4.2), the SFS will now select the best feature combination that it discovered by iterating from k=1 to max_k (forward), or max_k to min_k (backward). The size of the returned feature subset is then within max_k to min_k , depending on which combination scored best during cross validation. X.shape (150, 4) from mlxtend.feature_selection import SequentialFeatureSelector as SFS from sklearn.neighbors import KNeighborsClassifier from mlxtend.data import wine_data from sklearn.model_selection import train_test_split from sklearn.preprocessing import StandardScaler from sklearn.pipeline import make_pipeline X, y = wine_data() X_train, X_test, y_train, y_test= train_test_split(X, y, stratify=y, test_size=0.3, random_state=1) knn = KNeighborsClassifier(n_neighbors=2) sfs1 = SFS(estimator=knn, k_features=(3, 10), forward=True, floating=False, scoring='accuracy', cv=5) pipe = make_pipeline(StandardScaler(), sfs1) pipe.fit(X_train, y_train) print('best combination (ACC: %.3f): %s\\n' % (sfs1.k_score_, sfs1.k_feature_idx_)) print('all subsets:\\n', sfs1.subsets_) plot_sfs(sfs1.get_metric_dict(), kind='std_err'); best combination (ACC: 0.992): (0, 1, 2, 3, 6, 8, 9, 10, 11, 12) all subsets: {1: {'feature_idx': (6,), 'cv_scores': array([0.84 , 0.64 , 0.84 , 0.8 , 0.875]), 'avg_score': 0.799, 'feature_names': ('6',)}, 2: {'feature_idx': (6, 9), 'cv_scores': array([0.92 , 0.88 , 1. , 0.96 , 0.91666667]), 'avg_score': 0.9353333333333333, 'feature_names': ('6', '9')}, 3: {'feature_idx': (6, 9, 12), 'cv_scores': array([0.92 , 0.92 , 0.96 , 1. , 0.95833333]), 'avg_score': 0.9516666666666665, 'feature_names': ('6', '9', '12')}, 4: {'feature_idx': (3, 6, 9, 12), 'cv_scores': array([0.96 , 0.96 , 0.96 , 1. , 0.95833333]), 'avg_score': 0.9676666666666666, 'feature_names': ('3', '6', '9', '12')}, 5: {'feature_idx': (3, 6, 9, 10, 12), 'cv_scores': array([0.92, 0.96, 1. , 1. , 1. ]), 'avg_score': 0.976, 'feature_names': ('3', '6', '9', '10', '12')}, 6: {'feature_idx': (2, 3, 6, 9, 10, 12), 'cv_scores': array([0.92, 0.96, 1. , 0.96, 1. ]), 'avg_score': 0.968, 'feature_names': ('2', '3', '6', '9', '10', '12')}, 7: {'feature_idx': (0, 2, 3, 6, 9, 10, 12), 'cv_scores': array([0.92, 0.92, 1. , 1. , 1. ]), 'avg_score': 0.968, 'feature_names': ('0', '2', '3', '6', '9', '10', '12')}, 8: {'feature_idx': (0, 2, 3, 6, 8, 9, 10, 12), 'cv_scores': array([1. , 0.92, 1. , 1. , 1. ]), 'avg_score': 0.984, 'feature_names': ('0', '2', '3', '6', '8', '9', '10', '12')}, 9: {'feature_idx': (0, 2, 3, 6, 8, 9, 10, 11, 12), 'cv_scores': array([1. , 0.92, 1. , 1. , 1. ]), 'avg_score': 0.984, 'feature_names': ('0', '2', '3', '6', '8', '9', '10', '11', '12')}, 10: {'feature_idx': (0, 1, 2, 3, 6, 8, 9, 10, 11, 12), 'cv_scores': array([1. , 0.96, 1. , 1. , 1. ]), 'avg_score': 0.992, 'feature_names': ('0', '1', '2', '3', '6', '8', '9', '10', '11', '12')}} Example 10 -- Using other cross-validation schemes In addition to standard k-fold and stratified k-fold, other cross validation schemes can be used with SequentialFeatureSelector . For example, GroupKFold or LeaveOneOut cross-validation from scikit-learn. Using GroupKFold with SequentialFeatureSelector from mlxtend.feature_selection import SequentialFeatureSelector as SFS from sklearn.neighbors import KNeighborsClassifier from mlxtend.data import iris_data from sklearn.model_selection import GroupKFold import numpy as np X, y = iris_data() groups = np.arange(len(y)) // 10 print('groups: {}'.format(groups)) groups: [ 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 2 2 2 2 2 2 2 2 2 2 3 3 3 3 3 3 3 3 3 3 4 4 4 4 4 4 4 4 4 4 5 5 5 5 5 5 5 5 5 5 6 6 6 6 6 6 6 6 6 6 7 7 7 7 7 7 7 7 7 7 8 8 8 8 8 8 8 8 8 8 9 9 9 9 9 9 9 9 9 9 10 10 10 10 10 10 10 10 10 10 11 11 11 11 11 11 11 11 11 11 12 12 12 12 12 12 12 12 12 12 13 13 13 13 13 13 13 13 13 13 14 14 14 14 14 14 14 14 14 14] Calling the split() method of a scikit-learn cross-validator object will return a generator that yields train, test splits. cv_gen = GroupKFold(4).split(X, y, groups) cv_gen <generator object _BaseKFold.split at 0x17c109580> The cv parameter of SequentialFeatureSelector must be either an int or an iterable yielding train, test splits. This iterable can be constructed by passing the train, test split generator to the built-in list() function. cv = list(cv_gen) knn = KNeighborsClassifier(n_neighbors=2) sfs = SFS(estimator=knn, k_features=2, scoring='accuracy', cv=cv) sfs.fit(X, y) print('best combination (ACC: %.3f): %s\\n' % (sfs.k_score_, sfs.k_feature_idx_)) best combination (ACC: 0.940): (2, 3) Example 11 - Interrupting Long Runs for Intermediate Results If your run is taking too long, it is possible to trigger a KeyboardInterrupt (e.g., ctrl+c on a Mac, or interrupting the cell in a Jupyter notebook) to obtain temporary results. Toy dataset from sklearn.datasets import make_classification from sklearn.model_selection import train_test_split X, y = make_classification( n_samples=20000, n_features=500, n_informative=10, n_redundant=40, n_repeated=25, n_clusters_per_class=5, flip_y=0.05, class_sep=0.5, random_state=123, ) X_train, X_test, y_train, y_test = train_test_split( X, y, test_size=0.2, random_state=123 ) Long run with interruption from mlxtend.feature_selection import SequentialFeatureSelector as SFS from sklearn.linear_model import LogisticRegression model = LogisticRegression() sfs1 = SFS(model, k_features=10, forward=True, floating=False, verbose=2, scoring='accuracy', cv=5) sfs1 = sfs1.fit(X_train, y_train) [Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers. [Parallel(n_jobs=1)]: Done 1 out of 1 | elapsed: 0.0s remaining: 0.0s [Parallel(n_jobs=1)]: Done 500 out of 500 | elapsed: 8.3s finished [2023-05-17 08:36:32] Features: 1/10 -- score: 0.5965[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers. [Parallel(n_jobs=1)]: Done 1 out of 1 | elapsed: 0.0s remaining: 0.0s [Parallel(n_jobs=1)]: Done 499 out of 499 | elapsed: 13.8s finished [2023-05-17 08:36:45] Features: 2/10 -- score: 0.6256875000000001[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers. [Parallel(n_jobs=1)]: Done 1 out of 1 | elapsed: 0.0s remaining: 0.0s [Parallel(n_jobs=1)]: Done 498 out of 498 | elapsed: 18.1s finished [2023-05-17 08:37:03] Features: 3/10 -- score: 0.642[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers. [Parallel(n_jobs=1)]: Done 1 out of 1 | elapsed: 0.0s remaining: 0.0s [Parallel(n_jobs=1)]: Done 497 out of 497 | elapsed: 20.4s finished [2023-05-17 08:37:24] Features: 4/10 -- score: 0.6463125[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers. [Parallel(n_jobs=1)]: Done 1 out of 1 | elapsed: 0.0s remaining: 0.0s [Parallel(n_jobs=1)]: Done 496 out of 496 | elapsed: 22.2s finished [2023-05-17 08:37:46] Features: 5/10 -- score: 0.6495000000000001[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers. [Parallel(n_jobs=1)]: Done 1 out of 1 | elapsed: 0.1s remaining: 0.0s [Parallel(n_jobs=1)]: Done 495 out of 495 | elapsed: 26.1s finished [2023-05-17 08:38:12] Features: 6/10 -- score: 0.6514374999999999[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers. [Parallel(n_jobs=1)]: Done 1 out of 1 | elapsed: 0.0s remaining: 0.0s [Parallel(n_jobs=1)]: Done 494 out of 494 | elapsed: 26.1s finished [2023-05-17 08:38:38] Features: 7/10 -- score: 0.6533749999999999[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers. [Parallel(n_jobs=1)]: Done 1 out of 1 | elapsed: 0.0s remaining: 0.0s [Parallel(n_jobs=1)]: Done 493 out of 493 | elapsed: 25.3s finished [2023-05-17 08:39:04] Features: 8/10 -- score: 0.6545624999999999[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers. [Parallel(n_jobs=1)]: Done 1 out of 1 | elapsed: 0.1s remaining: 0.0s [Parallel(n_jobs=1)]: Done 492 out of 492 | elapsed: 26.3s finished [2023-05-17 08:39:30] Features: 9/10 -- score: 0.6549375[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers. [Parallel(n_jobs=1)]: Done 1 out of 1 | elapsed: 0.1s remaining: 0.0s [Parallel(n_jobs=1)]: Done 491 out of 491 | elapsed: 27.0s finished [2023-05-17 08:39:57] Features: 10/10 -- score: 0.6554374999999999 Finalizing the fit Note that the feature selection run hasn't finished, so certain attributes may not be available. In order to use the SFS instance, it is recommended to call finalize_fit , which will make SFS estimator appear as \"fitted\" process the temporary results: sfs1.finalize_fit() print(sfs1.k_feature_idx_) print(sfs1.k_score_) (30, 128, 144, 160, 184, 229, 256, 356, 439, 458) 0.6554374999999999 Example 12 - Using Pandas DataFrames Optionally, we can also use pandas DataFrames and pandas Series as input to the fit function. In this case, the column names of the pandas DataFrame will be used as feature names. However, note that if custom_feature_names are provided in the fit function, these custom_feature_names take precedence over the DataFrame column-based feature names. import pandas as pd from sklearn.neighbors import KNeighborsClassifier from sklearn.datasets import load_iris from mlxtend.feature_selection import SequentialFeatureSelector as SFS iris = load_iris() X = iris.data y = iris.target knn = KNeighborsClassifier(n_neighbors=4) sfs1 = SFS(knn, k_features=3, forward=True, floating=False, scoring='accuracy', cv=0) X_df = pd.DataFrame(X, columns=['sepal len', 'petal len', 'sepal width', 'petal width']) X_df.head() .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } sepal len petal len sepal width petal width 0 5.1 3.5 1.4 0.2 1 4.9 3.0 1.4 0.2 2 4.7 3.2 1.3 0.2 3 4.6 3.1 1.5 0.2 4 5.0 3.6 1.4 0.2 Also, the target array, y , can be optionally be cast as a Series: y_series = pd.Series(y) y_series.head() 0 0 1 0 2 0 3 0 4 0 dtype: int64 sfs1 = sfs1.fit(X_df, y_series) Note that the only difference of passing a pandas DataFrame as input is that the sfs1.subsets_ array will now contain a new column, sfs1.subsets_ {1: {'feature_idx': (3,), 'cv_scores': array([0.96]), 'avg_score': 0.96, 'feature_names': ('petal width',)}, 2: {'feature_idx': (2, 3), 'cv_scores': array([0.97333333]), 'avg_score': 0.9733333333333334, 'feature_names': ('sepal width', 'petal width')}, 3: {'feature_idx': (1, 2, 3), 'cv_scores': array([0.97333333]), 'avg_score': 0.9733333333333334, 'feature_names': ('petal len', 'sepal width', 'petal width')}} In mlxtend version >= 0.13 pandas DataFrames are supported as feature inputs to the SequentianFeatureSelector instead of NumPy arrays or other NumPy-like array types. Example 13 - Specifying Fixed Feature Sets Often, it may be useful to specify a fixed set of features we want to use for a given model (e.g., determined by prior knowledge or domain knowledge). Since MLxtend v 0.18.0, it is now possible to specify such features via the fixed_features attribute. This will mean that these features are guaranteed to be included in the selected subsets. Note that this feature works for all options regarding forward and backward selection, and using floating selection or not. The example below illustrates how we can set the features 0 and 2 in the dataset as fixed: from sklearn.neighbors import KNeighborsClassifier from sklearn.datasets import load_iris iris = load_iris() X = iris.data y = iris.target knn = KNeighborsClassifier(n_neighbors=3) from mlxtend.feature_selection import SequentialFeatureSelector as SFS sfs1 = SFS(knn, k_features=4, forward=True, floating=False, verbose=2, scoring='accuracy', fixed_features=(0, 2), cv=3) sfs1 = sfs1.fit(X, y) [Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers. [Parallel(n_jobs=1)]: Done 1 out of 1 | elapsed: 0.0s remaining: 0.0s [Parallel(n_jobs=1)]: Done 2 out of 2 | elapsed: 0.0s finished [2023-05-17 08:39:57] Features: 3/4 -- score: 0.9733333333333333[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers. [Parallel(n_jobs=1)]: Done 1 out of 1 | elapsed: 0.0s remaining: 0.0s [Parallel(n_jobs=1)]: Done 1 out of 1 | elapsed: 0.0s finished [2023-05-17 08:39:57] Features: 4/4 -- score: 0.9733333333333333 sfs1.subsets_ {2: {'feature_idx': (0, 2), 'cv_scores': array([0.98, 0.92, 0.94]), 'avg_score': 0.9466666666666667, 'feature_names': ('0', '2')}, 3: {'feature_idx': (0, 2, 3), 'cv_scores': array([0.98, 0.96, 0.98]), 'avg_score': 0.9733333333333333, 'feature_names': ('0', '2', '3')}, 4: {'feature_idx': (0, 1, 2, 3), 'cv_scores': array([0.98, 0.96, 0.98]), 'avg_score': 0.9733333333333333, 'feature_names': ('0', '1', '2', '3')}} If the input dataset is a pandas DataFrame, we can also use the column names directly: import pandas as pd X_df = pd.DataFrame(X, columns=['sepal len', 'petal len', 'sepal width', 'petal width']) X_df.head() .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } sepal len petal len sepal width petal width 0 5.1 3.5 1.4 0.2 1 4.9 3.0 1.4 0.2 2 4.7 3.2 1.3 0.2 3 4.6 3.1 1.5 0.2 4 5.0 3.6 1.4 0.2 sfs2 = SFS(knn, k_features=4, forward=True, floating=False, verbose=2, scoring='accuracy', fixed_features=('sepal len', 'petal len'), cv=3) sfs2 = sfs2.fit(X_df, y_series) [Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers. [Parallel(n_jobs=1)]: Done 1 out of 1 | elapsed: 0.0s remaining: 0.0s [Parallel(n_jobs=1)]: Done 2 out of 2 | elapsed: 0.0s finished [2023-05-17 08:39:57] Features: 3/4 -- score: 0.9466666666666667[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers. [Parallel(n_jobs=1)]: Done 1 out of 1 | elapsed: 0.0s remaining: 0.0s [Parallel(n_jobs=1)]: Done 1 out of 1 | elapsed: 0.0s finished [2023-05-17 08:39:57] Features: 4/4 -- score: 0.9733333333333333 sfs2.subsets_ {2: {'feature_idx': (0, 1), 'cv_scores': array([0.72, 0.74, 0.78]), 'avg_score': 0.7466666666666667, 'feature_names': ('sepal len', 'petal len')}, 3: {'feature_idx': (0, 1, 2), 'cv_scores': array([0.98, 0.92, 0.94]), 'avg_score': 0.9466666666666667, 'feature_names': ('sepal len', 'petal len', 'sepal width')}, 4: {'feature_idx': (0, 1, 2, 3), 'cv_scores': array([0.98, 0.96, 0.98]), 'avg_score': 0.9733333333333333, 'feature_names': ('sepal len', 'petal len', 'sepal width', 'petal width')}} Example 13 - Working with Feature Groups Since mlxtend v0.21.0, it is possible to specify feature groups. Feature groups allow you to group certain features together, such that they are always selected as a group. This can be very useful in contexts similar to one-hot encoding -- if you want to treat the one-hot encoded feature as a single feature: In the following example, we specify sepal length and sepal width as a feature group so that they are always selected together: from sklearn.datasets import load_iris import pandas as pd iris = load_iris() X = iris.data y = iris.target X_df = pd.DataFrame(X, columns=['sepal len', 'petal len', 'sepal wid', 'petal wid']) X_df.head() .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } sepal len petal len sepal wid petal wid 0 5.1 3.5 1.4 0.2 1 4.9 3.0 1.4 0.2 2 4.7 3.2 1.3 0.2 3 4.6 3.1 1.5 0.2 4 5.0 3.6 1.4 0.2 from sklearn.neighbors import KNeighborsClassifier from mlxtend.feature_selection import SequentialFeatureSelector as SFS knn = KNeighborsClassifier(n_neighbors=3) sfs1 = SFS(knn, k_features=2, scoring='accuracy', feature_groups=(['sepal len', 'sepal wid'], ['petal len'], ['petal wid']), cv=3) sfs1 = sfs1.fit(X_df, y) sfs1 = SFS(knn, k_features=2, scoring='accuracy', feature_groups=[[0, 2], [1], [3]], cv=3) sfs1 = sfs1.fit(X, y) Example 14 - Multiclass Metrics Certain scoring metrics like ROC AUC are originally designed for binary classification. However, they can also be used for multiclass settings. It is best to consult this scikit-learn metrics table for this. For example, we can use a ROC AUC One-Vs-Rest score via \u2018\"roc_auc_ovr\" as shown below. from sklearn.datasets import make_blobs X, y = make_blobs(n_samples=10, centers=4, n_features=5, random_state=0) from mlxtend.feature_selection import SequentialFeatureSelector as SFS sfs1 = SFS(knn, k_features=3, forward=True, floating=False, verbose=2, scoring='roc_auc_ovr', cv=0) sfs1 = sfs1.fit(X, y) [Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers. [Parallel(n_jobs=1)]: Done 1 out of 1 | elapsed: 0.0s remaining: 0.0s [Parallel(n_jobs=1)]: Done 5 out of 5 | elapsed: 0.0s finished [2023-05-17 08:39:57] Features: 1/3 -- score: 1.0[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers. [Parallel(n_jobs=1)]: Done 1 out of 1 | elapsed: 0.0s remaining: 0.0s [Parallel(n_jobs=1)]: Done 4 out of 4 | elapsed: 0.0s finished [2023-05-17 08:39:57] Features: 2/3 -- score: 1.0[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers. [Parallel(n_jobs=1)]: Done 1 out of 1 | elapsed: 0.0s remaining: 0.0s [Parallel(n_jobs=1)]: Done 3 out of 3 | elapsed: 0.0s finished [2023-05-17 08:39:57] Features: 3/3 -- score: 1.0 API SequentialFeatureSelector(estimator, k_features=1, forward=True, floating=False, verbose=0, scoring=None, cv=5, n_jobs=1, pre_dispatch='2 n_jobs', clone_estimator=True, fixed_features=None, feature_groups=None)* Sequential Feature Selection for Classification and Regression. Parameters estimator : scikit-learn classifier or regressor k_features : int or tuple or str (default: 1) Number of features to select, where k_features < the full feature set. New in 0.4.2: A tuple containing a min and max value can be provided, and the SFS will consider return any feature combination between min and max that scored highest in cross-validation. For example, the tuple (1, 4) will return any combination from 1 up to 4 features instead of a fixed number of features k. New in 0.8.0: A string argument \"best\" or \"parsimonious\". If \"best\" is provided, the feature selector will return the feature subset with the best cross-validation performance. If \"parsimonious\" is provided as an argument, the smallest feature subset that is within one standard error of the cross-validation performance will be selected. forward : bool (default: True) Forward selection if True, backward selection otherwise floating : bool (default: False) Adds a conditional exclusion/inclusion if True. verbose : int (default: 0), level of verbosity to use in logging. If 0, no output, if 1 number of features in current set, if 2 detailed logging i ncluding timestamp and cv scores at step. scoring : str, callable, or None (default: None) If None (default), uses 'accuracy' for sklearn classifiers and 'r2' for sklearn regressors. If str, uses a sklearn scoring metric string identifier, for example {accuracy, f1, precision, recall, roc_auc} for classifiers, {'mean_absolute_error', 'mean_squared_error'/'neg_mean_squared_error', 'median_absolute_error', 'r2'} for regressors. If a callable object or function is provided, it has to be conform with sklearn's signature scorer(estimator, X, y) ; see https://scikit-learn.org/stable/modules/generated/sklearn.metrics.make_scorer.html for more information. cv : int (default: 5) Integer or iterable yielding train, test splits. If cv is an integer and estimator is a classifier (or y consists of integer class labels) stratified k-fold. Otherwise regular k-fold cross-validation is performed. No cross-validation if cv is None, False, or 0. n_jobs : int (default: 1) The number of CPUs to use for evaluating different feature subsets in parallel. -1 means 'all CPUs'. pre_dispatch : int, or string (default: '2*n_jobs') Controls the number of jobs that get dispatched during parallel execution if n_jobs > 1 or n_jobs=-1 . Reducing this number can be useful to avoid an explosion of memory consumption when more jobs get dispatched than CPUs can process. This parameter can be: None, in which case all the jobs are immediately created and spawned. Use this for lightweight and fast-running jobs, to avoid delays due to on-demand spawning of the jobs An int, giving the exact number of total jobs that are spawned A string, giving an expression as a function of n_jobs, as in 2*n_jobs clone_estimator : bool (default: True) Clones estimator if True; works with the original estimator instance if False. Set to False if the estimator doesn't implement scikit-learn's set_params and get_params methods. In addition, it is required to set cv=0, and n_jobs=1. fixed_features : tuple (default: None) If not None , the feature indices provided as a tuple will be regarded as fixed by the feature selector. For example, if fixed_features=(1, 3, 7) , the 2nd, 4th, and 8th feature are guaranteed to be present in the solution. Note that if fixed_features is not None , make sure that the number of features to be selected is greater than len(fixed_features) . In other words, ensure that k_features > len(fixed_features) . New in mlxtend v. 0.18.0. feature_groups : list or None (default: None) Optional argument for treating certain features as a group. This means, the features within a group are always selected together, never split. For example, feature_groups=[[1], [2], [3, 4, 5]] specifies 3 feature groups. In this case, possible feature selection results with k_features=2 are [[1], [2] , [[1], [3, 4, 5]] , or [[2], [3, 4, 5]] . Feature groups can be useful for interpretability, for example, if features 3, 4, 5 are one-hot encoded features. (For more details, please read the notes at the bottom of this docstring). New in mlxtend v. 0.21.0. Attributes k_feature_idx_ : array-like, shape = [n_predictions] Feature Indices of the selected feature subsets. k_feature_names_ : array-like, shape = [n_predictions] Feature names of the selected feature subsets. If pandas DataFrames are used in the fit method, the feature names correspond to the column names. Otherwise, the feature names are string representation of the feature array indices. New in v 0.13.0. k_score_ : float Cross validation average score of the selected subset. subsets_ : dict A dictionary of selected feature subsets during the sequential selection, where the dictionary keys are the lengths k of these feature subsets. If the parameter feature_groups is not None, the value of key indicates the number of groups that are selected together. The dictionary values are dictionaries themselves with the following keys: 'feature_idx' (tuple of indices of the feature subset) 'feature_names' (tuple of feature names of the feat. subset) 'cv_scores' (list individual cross-validation scores) 'avg_score' (average cross-validation score) Note that if pandas DataFrames are used in the fit method, the 'feature_names' correspond to the column names. Otherwise, the feature names are string representation of the feature array indices. The 'feature_names' is new in v 0.13.0. Notes (1) If parameter feature_groups is not None, the number of features is equal to the number of feature groups, i.e. len(feature_groups) . For example, if feature_groups = [[0], [1], [2, 3], [4]] , then the max_features value cannot exceed 4. (2) Although two or more individual features may be considered as one group throughout the feature-selection process, it does not mean the individual features of that group have the same impact on the outcome. For instance, in linear regression, the coefficient of the feature 2 and 3 can be different even if they are considered as one group in feature_groups. (3) If both fixed_features and feature_groups are specified, ensure that each feature group contains the fixed_features selection. E.g., for a 3-feature set fixed_features=[0, 1] and feature_groups=[[0, 1], [2]] is valid; fixed_features=[0, 1] and feature_groups=[[0], [1, 2]] is not valid. (4) In case of KeyboardInterrupt, the dictionary subsets may not be completed. If user is still interested in getting the best score, they can use method `finalize_fit`. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/feature_selection/SequentialFeatureSelector/ Methods finalize_fit() None fit(X, y, groups=None, fit_params) Perform feature selection and learn model from training data. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. New in v 0.13.0: pandas DataFrames are now also accepted as argument for X. y : array-like, shape = [n_samples] Target values. New in v 0.13.0: pandas DataFrames are now also accepted as argument for y. groups : array-like, with shape (n_samples,), optional Group labels for the samples used while splitting the dataset into train/test set. Passed to the fit method of the cross-validator. fit_params : various, optional Additional parameters that are being passed to the estimator. For example, sample_weights=weights . Returns self : object fit_transform(X, y, groups=None, fit_params) Fit to training data then reduce X to its most important features. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. New in v 0.13.0: pandas DataFrames are now also accepted as argument for X. y : array-like, shape = [n_samples] Target values. New in v 0.13.0: a pandas Series are now also accepted as argument for y. groups : array-like, with shape (n_samples,), optional Group labels for the samples used while splitting the dataset into train/test set. Passed to the fit method of the cross-validator. fit_params : various, optional Additional parameters that are being passed to the estimator. For example, sample_weights=weights . Returns Reduced feature subset of X, shape={n_samples, k_features} generate_error_message_k_features(name) None get_metric_dict(confidence_interval=0.95) Return metric dictionary Parameters confidence_interval : float (default: 0.95) A positive float between 0.0 and 1.0 to compute the confidence interval bounds of the CV score averages. Returns Dictionary with items where each dictionary value is a list with the number of iterations (number of feature subsets) as its length. The dictionary keys corresponding to these lists are as follows: 'feature_idx': tuple of the indices of the feature subset 'cv_scores': list with individual CV scores 'avg_score': of CV average scores 'std_dev': standard deviation of the CV score average 'std_err': standard error of the CV score average 'ci_bound': confidence interval bound of the CV score average get_params(deep=True) Get parameters for this estimator. Parameters deep : bool, default=True If True, will return the parameters for this estimator and contained subobjects that are estimators. Returns params : dict Parameter names mapped to their values. set_params( params) Set the parameters of this estimator. Valid parameter keys can be listed with get_params() . Returns self transform(X) Reduce X to its most important features. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. New in v 0.13.0: pandas DataFrames are now also accepted as argument for X. Returns Reduced feature subset of X, shape={n_samples, k_features} Properties named_estimators Returns List of named estimator tuples, like [('svc', SVC(...))] ython","title":"SequentialFeatureSelector: The popular forward and backward feature selection approaches (including floating variants)"},{"location":"user_guide/feature_selection/SequentialFeatureSelector/#sequentialfeatureselector-the-popular-forward-and-backward-feature-selection-approaches-including-floating-variants","text":"Implementation of sequential feature algorithms (SFAs) -- greedy search algorithms -- that have been developed as a suboptimal solution to the computationally often not feasible exhaustive search. from mlxtend.feature_selection import SequentialFeatureSelector","title":"SequentialFeatureSelector: The popular forward and backward feature selection approaches (including floating variants)"},{"location":"user_guide/feature_selection/SequentialFeatureSelector/#overview","text":"Sequential feature selection algorithms are a family of greedy search algorithms that are used to reduce an initial d -dimensional feature space to a k -dimensional feature subspace where k < d . The motivation behind feature selection algorithms is to automatically select a subset of features most relevant to the problem. The goal of feature selection is two-fold: We want to improve the computational efficiency and reduce the model's generalization error by removing irrelevant features or noise. In addition, a wrapper approach such as sequential feature selection is advantageous if embedded feature selection -- for example, a regularization penalty like LASSO -- is not applicable. In a nutshell, SFAs remove or add one feature at a time based on the classifier performance until a feature subset of the desired size k is reached. There are four different flavors of SFAs available via the SequentialFeatureSelector : Sequential Forward Selection (SFS) Sequential Backward Selection (SBS) Sequential Forward Floating Selection (SFFS) Sequential Backward Floating Selection (SBFS) The floating variants, SFFS and SBFS, can be considered extensions to the simpler SFS and SBS algorithms. The floating algorithms have an additional exclusion or inclusion step to remove features once they were included (or excluded) so that a larger number of feature subset combinations can be sampled. It is important to emphasize that this step is conditional and only occurs if the resulting feature subset is assessed as \"better\" by the criterion function after the removal (or addition) of a particular feature. Furthermore, I added an optional check to skip the conditional exclusion steps if the algorithm gets stuck in cycles. How is this different from Recursive Feature Elimination (RFE) -- e.g., as implemented in sklearn.feature_selection.RFE ? RFE is computationally less complex using the feature weight coefficients (e.g., linear models) or feature importance (tree-based algorithms) to eliminate features recursively, whereas SFSs eliminate (or add) features based on a user-defined classifier/regression performance metric.","title":"Overview"},{"location":"user_guide/feature_selection/SequentialFeatureSelector/#tutorial-videos","text":"","title":"Tutorial Videos"},{"location":"user_guide/feature_selection/SequentialFeatureSelector/#visual-illustration","text":"A visual illustration of the sequential backward selection process is provided below, from the paper Joe Bemister-Buffington, Alex J. Wolf, Sebastian Raschka, and Leslie A. Kuhn (2020) Machine Learning to Identify Flexibility Signatures of Class A GPCR Inhibition Biomolecules 2020, 10, 454. https://www.mdpi.com/2218-273X/10/3/454#","title":"Visual Illustration"},{"location":"user_guide/feature_selection/SequentialFeatureSelector/#algorithmic-details","text":"","title":"Algorithmic Details"},{"location":"user_guide/feature_selection/SequentialFeatureSelector/#sequential-forward-selection-sfs","text":"Input: Y = \\{y_1, y_2, ..., y_d\\} The SFS algorithm takes the whole d -dimensional feature set as input. Output: X_k = \\{x_j \\; | \\;j = 1, 2, ..., k; \\; x_j \\in Y\\} , where k = (0, 1, 2, ..., d) SFS returns a subset of features; the number of selected features k , where k < d , has to be specified a priori . Initialization: X_0 = \\emptyset , k = 0 We initialize the algorithm with an empty set \\emptyset (\"null set\") so that k = 0 (where k is the size of the subset). Step 1 (Inclusion): x^+ = \\text{ arg max } J(X_k + x), \\text{ where } x \\in Y - X_k X_{k+1} = X_k + x^+ k = k + 1 Go to Step 1 in this step, we add an additional feature, x^+ , to our feature subset X_k . x^+ is the feature that maximizes our criterion function, that is, the feature that is associated with the best classifier performance if it is added to X_k . We repeat this procedure until the termination criterion is satisfied. Termination: k = p We add features from the feature subset X_k until the feature subset of size k contains the number of desired features p that we specified a priori .","title":"Sequential Forward Selection (SFS)"},{"location":"user_guide/feature_selection/SequentialFeatureSelector/#sequential-backward-selection-sbs","text":"Input: the set of all features, Y = \\{y_1, y_2, ..., y_d\\} The SBS algorithm takes the whole feature set as input. Output: X_k = \\{x_j \\; | \\;j = 1, 2, ..., k; \\; x_j \\in Y\\} , where k = (0, 1, 2, ..., d) SBS returns a subset of features; the number of selected features k , where k < d , has to be specified a priori . Initialization: X_0 = Y , k = d We initialize the algorithm with the given feature set so that the k = d . Step 1 (Exclusion): x^- = \\text{ arg max } J(X_k - x), \\text{ where } x \\in X_k X_{k-1} = X_k - x^- k = k - 1 Go to Step 1 In this step, we remove a feature, x^- from our feature subset X_k . x^- is the feature that maximizes our criterion function upon re,oval, that is, the feature that is associated with the best classifier performance if it is removed from X_k . We repeat this procedure until the termination criterion is satisfied. Termination: k = p We add features from the feature subset X_k until the feature subset of size k contains the number of desired features p that we specified a priori .","title":"Sequential Backward Selection (SBS)"},{"location":"user_guide/feature_selection/SequentialFeatureSelector/#sequential-backward-floating-selection-sbfs","text":"Input: the set of all features, Y = \\{y_1, y_2, ..., y_d\\} The SBFS algorithm takes the whole feature set as input. Output: X_k = \\{x_j \\; | \\;j = 1, 2, ..., k; \\; x_j \\in Y\\} , where k = (0, 1, 2, ..., d) SBFS returns a subset of features; the number of selected features k , where k < d , has to be specified a priori . Initialization: X_0 = Y , k = d We initialize the algorithm with the given feature set so that the k = d . Step 1 (Exclusion): x^- = \\text{ arg max } J(X_k - x), \\text{ where } x \\in X_k X_{k-1} = X_k - x^- k = k - 1 Go to Step 2 In this step, we remove a feature, x^- from our feature subset X_k . x^- is the feature that maximizes our criterion function upon removal, that is, the feature that is associated with the best classifier performance if it is removed from X_k . Step 2 (Conditional Inclusion): x^+ = \\text{ arg max } J(X_k + x), \\text{ where } x \\in Y - X_k if J(X_k + x) > J(X_k) : X_{k+1} = X_k + x^+ k = k + 1 Go to Step 1 In Step 2, we search for features that improve the classifier performance if they are added back to the feature subset. If such features exist, we add the feature x^+ for which the performance improvement is maximized. If k = 2 or an improvement cannot be made (i.e., such feature x^+ cannot be found), go back to step 1; else, repeat this step. Termination: k = p We add features from the feature subset X_k until the feature subset of size k contains the number of desired features p that we specified a priori .","title":"Sequential Backward Floating Selection (SBFS)"},{"location":"user_guide/feature_selection/SequentialFeatureSelector/#sequential-forward-floating-selection-sffs","text":"Input: the set of all features, Y = \\{y_1, y_2, ..., y_d\\} The SFFS algorithm takes the whole feature set as input, if our feature space consists of, e.g. 10, if our feature space consists of 10 dimensions ( d = 10 ). Output: a subset of features, X_k = \\{x_j \\; | \\;j = 1, 2, ..., k; \\; x_j \\in Y\\} , where k = (0, 1, 2, ..., d) The returned output of the algorithm is a subset of the feature space of a specified size. E.g., a subset of 5 features from a 10-dimensional feature space ( k = 5, d = 10 ). Initialization: X_0 = \\emptyset , k = 0 We initialize the algorithm with an empty set (\"null set\") so that the k = 0 (where k is the size of the subset) Step 1 (Inclusion): x^+ = \\text{ arg max } J(X_k + x), \\text{ where } x \\in Y - X_k X_{k+1} = X_k + x^+ k = k + 1 Go to Step 2 Step 2 (Conditional Exclusion): x^- = \\text{ arg max } J(X_k - x), \\text{ where } x \\in X_k if \\; J(X_k - x) > J(X_k) : X_{k-1} = X_k - x^- k = k - 1 Go to Step 1 In step 1, we include the feature from the feature space that leads to the best performance increase for our feature subset (assessed by the criterion function ). Then, we go over to step 2 In step 2, we only remove a feature if the resulting subset would gain an increase in performance. If k = 2 or an improvement cannot be made (i.e., such feature x^+ cannot be found), go back to step 1; else, repeat this step. Steps 1 and 2 are repeated until the Termination criterion is reached. Termination: stop when k equals the number of desired features","title":"Sequential Forward Floating Selection (SFFS)"},{"location":"user_guide/feature_selection/SequentialFeatureSelector/#references","text":"Ferri, F. J., Pudil P., Hatef, M., Kittler, J. (1994). \"Comparative study of techniques for large-scale feature selection.\" Pattern Recognition in Practice IV : 403-413. Pudil, P., Novovi\u010dov\u00e1, J., & Kittler, J. (1994). \"Floating search methods in feature selection.\" Pattern recognition letters 15.11 (1994): 1119-1125.","title":"References"},{"location":"user_guide/feature_selection/SequentialFeatureSelector/#example-1-a-simple-sequential-forward-selection-example","text":"Initializing a simple classifier from scikit-learn: from sklearn.neighbors import KNeighborsClassifier from sklearn.datasets import load_iris iris = load_iris() X = iris.data y = iris.target knn = KNeighborsClassifier(n_neighbors=4) We start by selection the \"best\" 3 features from the Iris dataset via Sequential Forward Selection (SFS). Here, we set forward=True and floating=False . By choosing cv=0 , we don't perform any cross-validation, therefore, the performance (here: 'accuracy' ) is computed entirely on the training set. from mlxtend.feature_selection import SequentialFeatureSelector as SFS sfs1 = SFS(knn, k_features=3, forward=True, floating=False, verbose=2, scoring='accuracy', cv=0) sfs1 = sfs1.fit(X, y) [Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers. [Parallel(n_jobs=1)]: Done 1 out of 1 | elapsed: 0.0s remaining: 0.0s [Parallel(n_jobs=1)]: Done 4 out of 4 | elapsed: 0.0s finished [2023-05-17 08:36:17] Features: 1/3 -- score: 0.96[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers. [Parallel(n_jobs=1)]: Done 1 out of 1 | elapsed: 0.0s remaining: 0.0s [Parallel(n_jobs=1)]: Done 3 out of 3 | elapsed: 0.0s finished [2023-05-17 08:36:17] Features: 2/3 -- score: 0.9733333333333334[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers. [Parallel(n_jobs=1)]: Done 1 out of 1 | elapsed: 0.0s remaining: 0.0s [Parallel(n_jobs=1)]: Done 2 out of 2 | elapsed: 0.0s finished [2023-05-17 08:36:17] Features: 3/3 -- score: 0.9733333333333334 Via the subsets_ attribute, we can take a look at the selected feature indices at each step: sfs1.subsets_ {1: {'feature_idx': (3,), 'cv_scores': array([0.96]), 'avg_score': 0.96, 'feature_names': ('3',)}, 2: {'feature_idx': (2, 3), 'cv_scores': array([0.97333333]), 'avg_score': 0.9733333333333334, 'feature_names': ('2', '3')}, 3: {'feature_idx': (1, 2, 3), 'cv_scores': array([0.97333333]), 'avg_score': 0.9733333333333334, 'feature_names': ('1', '2', '3')}} sfs1 = sfs1.fit(X, y) sfs1.subsets_ [Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers. [Parallel(n_jobs=1)]: Done 1 out of 1 | elapsed: 0.0s remaining: 0.0s [Parallel(n_jobs=1)]: Done 4 out of 4 | elapsed: 0.0s finished [2023-05-17 08:36:17] Features: 1/3 -- score: 0.96[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers. [Parallel(n_jobs=1)]: Done 1 out of 1 | elapsed: 0.0s remaining: 0.0s [Parallel(n_jobs=1)]: Done 3 out of 3 | elapsed: 0.0s finished [2023-05-17 08:36:17] Features: 2/3 -- score: 0.9733333333333334[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers. [Parallel(n_jobs=1)]: Done 1 out of 1 | elapsed: 0.0s remaining: 0.0s [Parallel(n_jobs=1)]: Done 2 out of 2 | elapsed: 0.0s finished [2023-05-17 08:36:17] Features: 3/3 -- score: 0.9733333333333334 {1: {'feature_idx': (3,), 'cv_scores': array([0.96]), 'avg_score': 0.96, 'feature_names': ('3',)}, 2: {'feature_idx': (2, 3), 'cv_scores': array([0.97333333]), 'avg_score': 0.9733333333333334, 'feature_names': ('2', '3')}, 3: {'feature_idx': (1, 2, 3), 'cv_scores': array([0.97333333]), 'avg_score': 0.9733333333333334, 'feature_names': ('1', '2', '3')}} Furthermore, we can access the indices of the 3 best features directly via the k_feature_idx_ attribute: sfs1.k_feature_idx_ (1, 2, 3) Finally, the prediction score for these 3 features can be accesses via k_score_ : sfs1.k_score_ 0.9733333333333334 Feature Names When working with large datasets, the feature indices might be hard to interpret. In this case, we recommend using pandas DataFrames with distinct column names as input: import pandas as pd df_X = pd.DataFrame(X, columns=[\"Sepal length\", \"Sepal width\", \"Petal length\", \"Petal width\"]) df_X.head() .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } Sepal length Sepal width Petal length Petal width 0 5.1 3.5 1.4 0.2 1 4.9 3.0 1.4 0.2 2 4.7 3.2 1.3 0.2 3 4.6 3.1 1.5 0.2 4 5.0 3.6 1.4 0.2 sfs1 = sfs1.fit(df_X, y) print('Best accuracy score: %.2f' % sfs1.k_score_) print('Best subset (indices):', sfs1.k_feature_idx_) print('Best subset (corresponding names):', sfs1.k_feature_names_) [Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers. Best accuracy score: 0.97 Best subset (indices): (1, 2, 3) Best subset (corresponding names): ('Sepal width', 'Petal length', 'Petal width') [Parallel(n_jobs=1)]: Done 1 out of 1 | elapsed: 0.0s remaining: 0.0s [Parallel(n_jobs=1)]: Done 4 out of 4 | elapsed: 0.0s finished [2023-05-17 08:36:17] Features: 1/3 -- score: 0.96[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers. [Parallel(n_jobs=1)]: Done 1 out of 1 | elapsed: 0.0s remaining: 0.0s [Parallel(n_jobs=1)]: Done 3 out of 3 | elapsed: 0.0s finished [2023-05-17 08:36:17] Features: 2/3 -- score: 0.9733333333333334[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers. [Parallel(n_jobs=1)]: Done 1 out of 1 | elapsed: 0.0s remaining: 0.0s [Parallel(n_jobs=1)]: Done 2 out of 2 | elapsed: 0.0s finished [2023-05-17 08:36:17] Features: 3/3 -- score: 0.9733333333333334","title":"Example 1 - A simple Sequential Forward Selection example"},{"location":"user_guide/feature_selection/SequentialFeatureSelector/#example-2-toggling-between-sfs-sbs-sffs-and-sbfs","text":"Using the forward and floating parameters, we can toggle between SFS, SBS, SFFS, and SBFS as shown below. Note that we are performing (stratified) 4-fold cross-validation for more robust estimates in contrast to Example 1. Via n_jobs=-1 , we choose to run the cross-validation on all our available CPU cores. # Sequential Forward Selection sfs = SFS(knn, k_features=3, forward=True, floating=False, scoring='accuracy', cv=4, n_jobs=-1) sfs = sfs.fit(X, y) print('\\nSequential Forward Selection (k=3):') print(sfs.k_feature_idx_) print('CV Score:') print(sfs.k_score_) ################################################### # Sequential Backward Selection sbs = SFS(knn, k_features=3, forward=False, floating=False, scoring='accuracy', cv=4, n_jobs=-1) sbs = sbs.fit(X, y) print('\\nSequential Backward Selection (k=3):') print(sbs.k_feature_idx_) print('CV Score:') print(sbs.k_score_) ################################################### # Sequential Forward Floating Selection sffs = SFS(knn, k_features=3, forward=True, floating=True, scoring='accuracy', cv=4, n_jobs=-1) sffs = sffs.fit(X, y) print('\\nSequential Forward Floating Selection (k=3):') print(sffs.k_feature_idx_) print('CV Score:') print(sffs.k_score_) ################################################### # Sequential Backward Floating Selection sbfs = SFS(knn, k_features=3, forward=False, floating=True, scoring='accuracy', cv=4, n_jobs=-1) sbfs = sbfs.fit(X, y) print('\\nSequential Backward Floating Selection (k=3):') print(sbfs.k_feature_idx_) print('CV Score:') print(sbfs.k_score_) Sequential Forward Selection (k=3): (1, 2, 3) CV Score: 0.9731507823613088 Sequential Backward Selection (k=3): (1, 2, 3) CV Score: 0.9731507823613088 Sequential Forward Floating Selection (k=3): (1, 2, 3) CV Score: 0.9731507823613088 Sequential Backward Floating Selection (k=3): (1, 2, 3) CV Score: 0.9731507823613088 In this simple scenario, selecting the best 3 features out of the 4 available features in the Iris set, we end up with similar results regardless of which sequential selection algorithms we used.","title":"Example 2 - Toggling between SFS, SBS, SFFS, and SBFS"},{"location":"user_guide/feature_selection/SequentialFeatureSelector/#example-3-visualizing-the-results-in-dataframes","text":"For our convenience, we can visualize the output from the feature selection in a pandas DataFrame format using the get_metric_dict method of the SequentialFeatureSelector object. The columns std_dev and std_err represent the standard deviation and standard errors of the cross-validation scores, respectively. Below, we see the DataFrame of the Sequential Forward Selector from Example 2: import pandas as pd pd.DataFrame.from_dict(sfs.get_metric_dict()).T .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } feature_idx cv_scores avg_score feature_names ci_bound std_dev std_err 1 (3,) [0.9736842105263158, 0.9473684210526315, 0.918... 0.959993 (3,) 0.048319 0.030143 0.017403 2 (2, 3) [0.9736842105263158, 0.9473684210526315, 0.918... 0.959993 (2, 3) 0.048319 0.030143 0.017403 3 (1, 2, 3) [0.9736842105263158, 1.0, 0.9459459459459459, ... 0.973151 (1, 2, 3) 0.030639 0.019113 0.011035 Now, let's compare it to the Sequential Backward Selector: pd.DataFrame.from_dict(sbs.get_metric_dict()).T .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } feature_idx cv_scores avg_score feature_names ci_bound std_dev std_err 4 (0, 1, 2, 3) [0.9736842105263158, 0.9473684210526315, 0.918... 0.953236 (0, 1, 2, 3) 0.03602 0.022471 0.012974 3 (1, 2, 3) [0.9736842105263158, 1.0, 0.9459459459459459, ... 0.973151 (1, 2, 3) 0.030639 0.019113 0.011035 We can see that both SFS and SBFS found the same \"best\" 3 features, however, the intermediate steps where obviously different. The ci_bound column in the DataFrames above represents the confidence interval around the computed cross-validation scores. By default, a confidence interval of 95% is used, but we can use different confidence bounds via the confidence_interval parameter. E.g., the confidence bounds for a 90% confidence interval can be obtained as follows: pd.DataFrame.from_dict(sbs.get_metric_dict(confidence_interval=0.90)).T .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } feature_idx cv_scores avg_score feature_names ci_bound std_dev std_err 4 (0, 1, 2, 3) [0.9736842105263158, 0.9473684210526315, 0.918... 0.953236 (0, 1, 2, 3) 0.027658 0.022471 0.012974 3 (1, 2, 3) [0.9736842105263158, 1.0, 0.9459459459459459, ... 0.973151 (1, 2, 3) 0.023525 0.019113 0.011035","title":"Example 3 - Visualizing the results in DataFrames"},{"location":"user_guide/feature_selection/SequentialFeatureSelector/#example-4-plotting-the-results","text":"After importing the little helper function plotting.plot_sequential_feature_selection , we can also visualize the results using matplotlib figures. from mlxtend.plotting import plot_sequential_feature_selection as plot_sfs import matplotlib.pyplot as plt sfs = SFS(knn, k_features=4, forward=True, floating=False, scoring='accuracy', verbose=2, cv=5) sfs = sfs.fit(X, y) fig1 = plot_sfs(sfs.get_metric_dict(), kind='std_dev') plt.ylim([0.8, 1]) plt.title('Sequential Forward Selection (w. StdDev)') plt.grid() plt.show() [Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers. [Parallel(n_jobs=1)]: Done 1 out of 1 | elapsed: 0.0s remaining: 0.0s [Parallel(n_jobs=1)]: Done 4 out of 4 | elapsed: 0.0s finished [2023-05-17 08:36:18] Features: 1/4 -- score: 0.96[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers. [Parallel(n_jobs=1)]: Done 1 out of 1 | elapsed: 0.0s remaining: 0.0s [Parallel(n_jobs=1)]: Done 3 out of 3 | elapsed: 0.0s finished [2023-05-17 08:36:18] Features: 2/4 -- score: 0.9666666666666668[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers. [Parallel(n_jobs=1)]: Done 1 out of 1 | elapsed: 0.0s remaining: 0.0s [Parallel(n_jobs=1)]: Done 2 out of 2 | elapsed: 0.0s finished [2023-05-17 08:36:18] Features: 3/4 -- score: 0.9533333333333334[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers. [Parallel(n_jobs=1)]: Done 1 out of 1 | elapsed: 0.0s remaining: 0.0s [Parallel(n_jobs=1)]: Done 1 out of 1 | elapsed: 0.0s finished [2023-05-17 08:36:18] Features: 4/4 -- score: 0.9733333333333334","title":"Example 4 - Plotting the results"},{"location":"user_guide/feature_selection/SequentialFeatureSelector/#example-5-sequential-feature-selection-for-regression","text":"Similar to the classification examples above, the SequentialFeatureSelector also supports scikit-learn's estimators for regression. from sklearn.linear_model import LinearRegression from sklearn.datasets import fetch_california_housing data = fetch_california_housing() X, y = data.data, data.target lr = LinearRegression() sfs = SFS(lr, k_features=8, forward=True, floating=False, scoring='neg_mean_squared_error', cv=10) sfs = sfs.fit(X, y) fig = plot_sfs(sfs.get_metric_dict(), kind='std_err') plt.title('Sequential Forward Selection (w. StdErr)') plt.grid() plt.show()","title":"Example 5 - Sequential Feature Selection for Regression"},{"location":"user_guide/feature_selection/SequentialFeatureSelector/#example-6-feature-selection-with-fixed-trainvalidation-splits","text":"If you do not wish to use cross-validation (here: k-fold cross-validation, i.e., rotating training and validation folds), you can use the PredefinedHoldoutSplit class to specify your own, fixed training and validation split. from sklearn.datasets import load_iris from mlxtend.evaluate import PredefinedHoldoutSplit import numpy as np iris = load_iris() X = iris.data y = iris.target rng = np.random.RandomState(123) my_validation_indices = rng.permutation(np.arange(150))[:30] print(my_validation_indices) [ 72 112 132 88 37 138 87 42 8 90 141 33 59 116 135 104 36 13 63 45 28 133 24 127 46 20 31 121 117 4] from sklearn.neighbors import KNeighborsClassifier from mlxtend.feature_selection import SequentialFeatureSelector as SFS knn = KNeighborsClassifier(n_neighbors=4) piter = PredefinedHoldoutSplit(my_validation_indices) sfs1 = SFS(knn, k_features=3, forward=True, floating=False, verbose=2, scoring='accuracy', cv=piter) sfs1 = sfs1.fit(X, y) [Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers. [Parallel(n_jobs=1)]: Done 1 out of 1 | elapsed: 0.0s remaining: 0.0s [Parallel(n_jobs=1)]: Done 4 out of 4 | elapsed: 0.0s finished [2023-05-17 08:36:19] Features: 1/3 -- score: 0.9666666666666667[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers. [Parallel(n_jobs=1)]: Done 1 out of 1 | elapsed: 0.0s remaining: 0.0s [Parallel(n_jobs=1)]: Done 3 out of 3 | elapsed: 0.0s finished [2023-05-17 08:36:19] Features: 2/3 -- score: 0.9666666666666667[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers. [Parallel(n_jobs=1)]: Done 1 out of 1 | elapsed: 0.0s remaining: 0.0s [Parallel(n_jobs=1)]: Done 2 out of 2 | elapsed: 0.0s finished [2023-05-17 08:36:19] Features: 3/3 -- score: 0.9666666666666667","title":"Example 6 -- Feature Selection with Fixed Train/Validation Splits"},{"location":"user_guide/feature_selection/SequentialFeatureSelector/#example-7-using-the-selected-feature-subset-for-making-new-predictions","text":"# Initialize the dataset from sklearn.neighbors import KNeighborsClassifier from sklearn.datasets import load_iris from sklearn.model_selection import train_test_split iris = load_iris() X, y = iris.data, iris.target X_train, X_test, y_train, y_test = train_test_split( X, y, test_size=0.33, random_state=1) knn = KNeighborsClassifier(n_neighbors=4) # Select the \"best\" three features via # 5-fold cross-validation on the training set. from mlxtend.feature_selection import SequentialFeatureSelector as SFS sfs1 = SFS(knn, k_features=3, forward=True, floating=False, scoring='accuracy', cv=5) sfs1 = sfs1.fit(X_train, y_train) print('Selected features:', sfs1.k_feature_idx_) Selected features: (1, 2, 3) # Generate the new subsets based on the selected features # Note that the transform call is equivalent to # X_train[:, sfs1.k_feature_idx_] X_train_sfs = sfs1.transform(X_train) X_test_sfs = sfs1.transform(X_test) # Fit the estimator using the new feature subset # and make a prediction on the test data knn.fit(X_train_sfs, y_train) y_pred = knn.predict(X_test_sfs) # Compute the accuracy of the prediction acc = float((y_test == y_pred).sum()) / y_pred.shape[0] print('Test set accuracy: %.2f %%' % (acc * 100)) Test set accuracy: 96.00 %","title":"Example 7 -- Using the Selected Feature Subset For Making New Predictions"},{"location":"user_guide/feature_selection/SequentialFeatureSelector/#example-8-sequential-feature-selection-and-gridsearch","text":"In the following example, we are tuning the SFS's estimator using GridSearch. To avoid unwanted behavior or side-effects, it's recommended to use the estimator inside and outside of SFS as separate instances. # Initialize the dataset from sklearn.neighbors import KNeighborsClassifier from sklearn.datasets import load_iris from sklearn.model_selection import train_test_split iris = load_iris() X, y = iris.data, iris.target X_train, X_test, y_train, y_test = train_test_split( X, y, test_size=0.2, random_state=123) from sklearn.model_selection import GridSearchCV from sklearn.pipeline import Pipeline from mlxtend.feature_selection import SequentialFeatureSelector as SFS import mlxtend knn1 = KNeighborsClassifier() knn2 = KNeighborsClassifier() sfs1 = SFS(estimator=knn1, k_features=3, forward=True, floating=False, scoring='accuracy', cv=5) pipe = Pipeline([('sfs', sfs1), ('knn2', knn2)]) param_grid = { 'sfs__k_features': [1, 2, 3], 'sfs__estimator__n_neighbors': [3, 4, 7], # inner knn 'knn2__n_neighbors': [3, 4, 7] # outer knn } gs = GridSearchCV(estimator=pipe, param_grid=param_grid, scoring='accuracy', n_jobs=1, cv=5, refit=False) # run gridearch gs = gs.fit(X_train, y_train) Let's take a look at the suggested hyperparameters below: for i in range(len(gs.cv_results_['params'])): print(gs.cv_results_['params'][i], 'test acc.:', gs.cv_results_['mean_test_score'][i]) The \"best\" parameters determined by GridSearch are ... print(\"Best parameters via GridSearch\", gs.best_params_) Best parameters via GridSearch {'knn2__n_neighbors': 7, 'sfs__estimator__n_neighbors': 3, 'sfs__k_features': 3} pipe.set_params(**gs.best_params_).fit(X_train, y_train) #sk-container-id-1 {color: black;background-color: white;}#sk-container-id-1 pre{padding: 0;}#sk-container-id-1 div.sk-toggleable {background-color: white;}#sk-container-id-1 label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.3em;box-sizing: border-box;text-align: center;}#sk-container-id-1 label.sk-toggleable__label-arrow:before {content: \"\u25b8\";float: left;margin-right: 0.25em;color: #696969;}#sk-container-id-1 label.sk-toggleable__label-arrow:hover:before {color: black;}#sk-container-id-1 div.sk-estimator:hover label.sk-toggleable__label-arrow:before {color: black;}#sk-container-id-1 div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}#sk-container-id-1 div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}#sk-container-id-1 input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}#sk-container-id-1 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {content: \"\u25be\";}#sk-container-id-1 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}#sk-container-id-1 div.sk-estimator {font-family: monospace;background-color: #f0f8ff;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;margin-bottom: 0.5em;}#sk-container-id-1 div.sk-estimator:hover {background-color: #d4ebff;}#sk-container-id-1 div.sk-parallel-item::after {content: \"\";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}#sk-container-id-1 div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 div.sk-serial::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: 0;}#sk-container-id-1 div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;padding-right: 0.2em;padding-left: 0.2em;position: relative;}#sk-container-id-1 div.sk-item {position: relative;z-index: 1;}#sk-container-id-1 div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;position: relative;}#sk-container-id-1 div.sk-item::before, #sk-container-id-1 div.sk-parallel-item::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: -1;}#sk-container-id-1 div.sk-parallel-item {display: flex;flex-direction: column;z-index: 1;position: relative;background-color: white;}#sk-container-id-1 div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}#sk-container-id-1 div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}#sk-container-id-1 div.sk-parallel-item:only-child::after {width: 0;}#sk-container-id-1 div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0 0.4em 0.5em 0.4em;box-sizing: border-box;padding-bottom: 0.4em;background-color: white;}#sk-container-id-1 div.sk-label label {font-family: monospace;font-weight: bold;display: inline-block;line-height: 1.2em;}#sk-container-id-1 div.sk-label-container {text-align: center;}#sk-container-id-1 div.sk-container {/* jupyter's `normalize.less` sets `[hidden] { display: none; }` but bootstrap.min.css set `[hidden] { display: none !important; }` so we also need the `!important` here to be able to override the default hidden behavior on the sphinx rendered scikit-learn.org. See: https://github.com/scikit-learn/scikit-learn/issues/21755 */display: inline-block !important;position: relative;}#sk-container-id-1 div.sk-text-repr-fallback {display: none;} Pipeline(steps=[('sfs', SequentialFeatureSelector(estimator=KNeighborsClassifier(n_neighbors=3), k_features=(3, 3), scoring='accuracy')), ('knn2', KNeighborsClassifier(n_neighbors=7))]) In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org. Pipeline Pipeline(steps=[('sfs', SequentialFeatureSelector(estimator=KNeighborsClassifier(n_neighbors=3), k_features=(3, 3), scoring='accuracy')), ('knn2', KNeighborsClassifier(n_neighbors=7))]) sfs: SequentialFeatureSelector SequentialFeatureSelector(estimator=KNeighborsClassifier(n_neighbors=3), k_features=(3, 3), scoring='accuracy') estimator: KNeighborsClassifier KNeighborsClassifier(n_neighbors=3) KNeighborsClassifier KNeighborsClassifier(n_neighbors=3) KNeighborsClassifier KNeighborsClassifier(n_neighbors=7)","title":"Example 8 -- Sequential Feature Selection and GridSearch"},{"location":"user_guide/feature_selection/SequentialFeatureSelector/#example-9-selecting-the-best-feature-combination-in-a-k-range","text":"If k_features is set to to a tuple (min_k, max_k) (new in 0.4.2), the SFS will now select the best feature combination that it discovered by iterating from k=1 to max_k (forward), or max_k to min_k (backward). The size of the returned feature subset is then within max_k to min_k , depending on which combination scored best during cross validation. X.shape (150, 4) from mlxtend.feature_selection import SequentialFeatureSelector as SFS from sklearn.neighbors import KNeighborsClassifier from mlxtend.data import wine_data from sklearn.model_selection import train_test_split from sklearn.preprocessing import StandardScaler from sklearn.pipeline import make_pipeline X, y = wine_data() X_train, X_test, y_train, y_test= train_test_split(X, y, stratify=y, test_size=0.3, random_state=1) knn = KNeighborsClassifier(n_neighbors=2) sfs1 = SFS(estimator=knn, k_features=(3, 10), forward=True, floating=False, scoring='accuracy', cv=5) pipe = make_pipeline(StandardScaler(), sfs1) pipe.fit(X_train, y_train) print('best combination (ACC: %.3f): %s\\n' % (sfs1.k_score_, sfs1.k_feature_idx_)) print('all subsets:\\n', sfs1.subsets_) plot_sfs(sfs1.get_metric_dict(), kind='std_err'); best combination (ACC: 0.992): (0, 1, 2, 3, 6, 8, 9, 10, 11, 12) all subsets: {1: {'feature_idx': (6,), 'cv_scores': array([0.84 , 0.64 , 0.84 , 0.8 , 0.875]), 'avg_score': 0.799, 'feature_names': ('6',)}, 2: {'feature_idx': (6, 9), 'cv_scores': array([0.92 , 0.88 , 1. , 0.96 , 0.91666667]), 'avg_score': 0.9353333333333333, 'feature_names': ('6', '9')}, 3: {'feature_idx': (6, 9, 12), 'cv_scores': array([0.92 , 0.92 , 0.96 , 1. , 0.95833333]), 'avg_score': 0.9516666666666665, 'feature_names': ('6', '9', '12')}, 4: {'feature_idx': (3, 6, 9, 12), 'cv_scores': array([0.96 , 0.96 , 0.96 , 1. , 0.95833333]), 'avg_score': 0.9676666666666666, 'feature_names': ('3', '6', '9', '12')}, 5: {'feature_idx': (3, 6, 9, 10, 12), 'cv_scores': array([0.92, 0.96, 1. , 1. , 1. ]), 'avg_score': 0.976, 'feature_names': ('3', '6', '9', '10', '12')}, 6: {'feature_idx': (2, 3, 6, 9, 10, 12), 'cv_scores': array([0.92, 0.96, 1. , 0.96, 1. ]), 'avg_score': 0.968, 'feature_names': ('2', '3', '6', '9', '10', '12')}, 7: {'feature_idx': (0, 2, 3, 6, 9, 10, 12), 'cv_scores': array([0.92, 0.92, 1. , 1. , 1. ]), 'avg_score': 0.968, 'feature_names': ('0', '2', '3', '6', '9', '10', '12')}, 8: {'feature_idx': (0, 2, 3, 6, 8, 9, 10, 12), 'cv_scores': array([1. , 0.92, 1. , 1. , 1. ]), 'avg_score': 0.984, 'feature_names': ('0', '2', '3', '6', '8', '9', '10', '12')}, 9: {'feature_idx': (0, 2, 3, 6, 8, 9, 10, 11, 12), 'cv_scores': array([1. , 0.92, 1. , 1. , 1. ]), 'avg_score': 0.984, 'feature_names': ('0', '2', '3', '6', '8', '9', '10', '11', '12')}, 10: {'feature_idx': (0, 1, 2, 3, 6, 8, 9, 10, 11, 12), 'cv_scores': array([1. , 0.96, 1. , 1. , 1. ]), 'avg_score': 0.992, 'feature_names': ('0', '1', '2', '3', '6', '8', '9', '10', '11', '12')}}","title":"Example 9 -- Selecting the \"best\"  feature combination in a k-range"},{"location":"user_guide/feature_selection/SequentialFeatureSelector/#example-10-using-other-cross-validation-schemes","text":"In addition to standard k-fold and stratified k-fold, other cross validation schemes can be used with SequentialFeatureSelector . For example, GroupKFold or LeaveOneOut cross-validation from scikit-learn.","title":"Example 10 -- Using other cross-validation schemes"},{"location":"user_guide/feature_selection/SequentialFeatureSelector/#using-groupkfold-with-sequentialfeatureselector","text":"from mlxtend.feature_selection import SequentialFeatureSelector as SFS from sklearn.neighbors import KNeighborsClassifier from mlxtend.data import iris_data from sklearn.model_selection import GroupKFold import numpy as np X, y = iris_data() groups = np.arange(len(y)) // 10 print('groups: {}'.format(groups)) groups: [ 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 2 2 2 2 2 2 2 2 2 2 3 3 3 3 3 3 3 3 3 3 4 4 4 4 4 4 4 4 4 4 5 5 5 5 5 5 5 5 5 5 6 6 6 6 6 6 6 6 6 6 7 7 7 7 7 7 7 7 7 7 8 8 8 8 8 8 8 8 8 8 9 9 9 9 9 9 9 9 9 9 10 10 10 10 10 10 10 10 10 10 11 11 11 11 11 11 11 11 11 11 12 12 12 12 12 12 12 12 12 12 13 13 13 13 13 13 13 13 13 13 14 14 14 14 14 14 14 14 14 14] Calling the split() method of a scikit-learn cross-validator object will return a generator that yields train, test splits. cv_gen = GroupKFold(4).split(X, y, groups) cv_gen <generator object _BaseKFold.split at 0x17c109580> The cv parameter of SequentialFeatureSelector must be either an int or an iterable yielding train, test splits. This iterable can be constructed by passing the train, test split generator to the built-in list() function. cv = list(cv_gen) knn = KNeighborsClassifier(n_neighbors=2) sfs = SFS(estimator=knn, k_features=2, scoring='accuracy', cv=cv) sfs.fit(X, y) print('best combination (ACC: %.3f): %s\\n' % (sfs.k_score_, sfs.k_feature_idx_)) best combination (ACC: 0.940): (2, 3)","title":"Using GroupKFold with SequentialFeatureSelector"},{"location":"user_guide/feature_selection/SequentialFeatureSelector/#example-11-interrupting-long-runs-for-intermediate-results","text":"If your run is taking too long, it is possible to trigger a KeyboardInterrupt (e.g., ctrl+c on a Mac, or interrupting the cell in a Jupyter notebook) to obtain temporary results. Toy dataset from sklearn.datasets import make_classification from sklearn.model_selection import train_test_split X, y = make_classification( n_samples=20000, n_features=500, n_informative=10, n_redundant=40, n_repeated=25, n_clusters_per_class=5, flip_y=0.05, class_sep=0.5, random_state=123, ) X_train, X_test, y_train, y_test = train_test_split( X, y, test_size=0.2, random_state=123 ) Long run with interruption from mlxtend.feature_selection import SequentialFeatureSelector as SFS from sklearn.linear_model import LogisticRegression model = LogisticRegression() sfs1 = SFS(model, k_features=10, forward=True, floating=False, verbose=2, scoring='accuracy', cv=5) sfs1 = sfs1.fit(X_train, y_train) [Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers. [Parallel(n_jobs=1)]: Done 1 out of 1 | elapsed: 0.0s remaining: 0.0s [Parallel(n_jobs=1)]: Done 500 out of 500 | elapsed: 8.3s finished [2023-05-17 08:36:32] Features: 1/10 -- score: 0.5965[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers. [Parallel(n_jobs=1)]: Done 1 out of 1 | elapsed: 0.0s remaining: 0.0s [Parallel(n_jobs=1)]: Done 499 out of 499 | elapsed: 13.8s finished [2023-05-17 08:36:45] Features: 2/10 -- score: 0.6256875000000001[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers. [Parallel(n_jobs=1)]: Done 1 out of 1 | elapsed: 0.0s remaining: 0.0s [Parallel(n_jobs=1)]: Done 498 out of 498 | elapsed: 18.1s finished [2023-05-17 08:37:03] Features: 3/10 -- score: 0.642[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers. [Parallel(n_jobs=1)]: Done 1 out of 1 | elapsed: 0.0s remaining: 0.0s [Parallel(n_jobs=1)]: Done 497 out of 497 | elapsed: 20.4s finished [2023-05-17 08:37:24] Features: 4/10 -- score: 0.6463125[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers. [Parallel(n_jobs=1)]: Done 1 out of 1 | elapsed: 0.0s remaining: 0.0s [Parallel(n_jobs=1)]: Done 496 out of 496 | elapsed: 22.2s finished [2023-05-17 08:37:46] Features: 5/10 -- score: 0.6495000000000001[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers. [Parallel(n_jobs=1)]: Done 1 out of 1 | elapsed: 0.1s remaining: 0.0s [Parallel(n_jobs=1)]: Done 495 out of 495 | elapsed: 26.1s finished [2023-05-17 08:38:12] Features: 6/10 -- score: 0.6514374999999999[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers. [Parallel(n_jobs=1)]: Done 1 out of 1 | elapsed: 0.0s remaining: 0.0s [Parallel(n_jobs=1)]: Done 494 out of 494 | elapsed: 26.1s finished [2023-05-17 08:38:38] Features: 7/10 -- score: 0.6533749999999999[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers. [Parallel(n_jobs=1)]: Done 1 out of 1 | elapsed: 0.0s remaining: 0.0s [Parallel(n_jobs=1)]: Done 493 out of 493 | elapsed: 25.3s finished [2023-05-17 08:39:04] Features: 8/10 -- score: 0.6545624999999999[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers. [Parallel(n_jobs=1)]: Done 1 out of 1 | elapsed: 0.1s remaining: 0.0s [Parallel(n_jobs=1)]: Done 492 out of 492 | elapsed: 26.3s finished [2023-05-17 08:39:30] Features: 9/10 -- score: 0.6549375[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers. [Parallel(n_jobs=1)]: Done 1 out of 1 | elapsed: 0.1s remaining: 0.0s [Parallel(n_jobs=1)]: Done 491 out of 491 | elapsed: 27.0s finished [2023-05-17 08:39:57] Features: 10/10 -- score: 0.6554374999999999 Finalizing the fit Note that the feature selection run hasn't finished, so certain attributes may not be available. In order to use the SFS instance, it is recommended to call finalize_fit , which will make SFS estimator appear as \"fitted\" process the temporary results: sfs1.finalize_fit() print(sfs1.k_feature_idx_) print(sfs1.k_score_) (30, 128, 144, 160, 184, 229, 256, 356, 439, 458) 0.6554374999999999","title":"Example 11 - Interrupting Long Runs for Intermediate Results"},{"location":"user_guide/feature_selection/SequentialFeatureSelector/#example-12-using-pandas-dataframes","text":"Optionally, we can also use pandas DataFrames and pandas Series as input to the fit function. In this case, the column names of the pandas DataFrame will be used as feature names. However, note that if custom_feature_names are provided in the fit function, these custom_feature_names take precedence over the DataFrame column-based feature names. import pandas as pd from sklearn.neighbors import KNeighborsClassifier from sklearn.datasets import load_iris from mlxtend.feature_selection import SequentialFeatureSelector as SFS iris = load_iris() X = iris.data y = iris.target knn = KNeighborsClassifier(n_neighbors=4) sfs1 = SFS(knn, k_features=3, forward=True, floating=False, scoring='accuracy', cv=0) X_df = pd.DataFrame(X, columns=['sepal len', 'petal len', 'sepal width', 'petal width']) X_df.head() .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } sepal len petal len sepal width petal width 0 5.1 3.5 1.4 0.2 1 4.9 3.0 1.4 0.2 2 4.7 3.2 1.3 0.2 3 4.6 3.1 1.5 0.2 4 5.0 3.6 1.4 0.2 Also, the target array, y , can be optionally be cast as a Series: y_series = pd.Series(y) y_series.head() 0 0 1 0 2 0 3 0 4 0 dtype: int64 sfs1 = sfs1.fit(X_df, y_series) Note that the only difference of passing a pandas DataFrame as input is that the sfs1.subsets_ array will now contain a new column, sfs1.subsets_ {1: {'feature_idx': (3,), 'cv_scores': array([0.96]), 'avg_score': 0.96, 'feature_names': ('petal width',)}, 2: {'feature_idx': (2, 3), 'cv_scores': array([0.97333333]), 'avg_score': 0.9733333333333334, 'feature_names': ('sepal width', 'petal width')}, 3: {'feature_idx': (1, 2, 3), 'cv_scores': array([0.97333333]), 'avg_score': 0.9733333333333334, 'feature_names': ('petal len', 'sepal width', 'petal width')}} In mlxtend version >= 0.13 pandas DataFrames are supported as feature inputs to the SequentianFeatureSelector instead of NumPy arrays or other NumPy-like array types.","title":"Example 12 - Using Pandas DataFrames"},{"location":"user_guide/feature_selection/SequentialFeatureSelector/#example-13-specifying-fixed-feature-sets","text":"Often, it may be useful to specify a fixed set of features we want to use for a given model (e.g., determined by prior knowledge or domain knowledge). Since MLxtend v 0.18.0, it is now possible to specify such features via the fixed_features attribute. This will mean that these features are guaranteed to be included in the selected subsets. Note that this feature works for all options regarding forward and backward selection, and using floating selection or not. The example below illustrates how we can set the features 0 and 2 in the dataset as fixed: from sklearn.neighbors import KNeighborsClassifier from sklearn.datasets import load_iris iris = load_iris() X = iris.data y = iris.target knn = KNeighborsClassifier(n_neighbors=3) from mlxtend.feature_selection import SequentialFeatureSelector as SFS sfs1 = SFS(knn, k_features=4, forward=True, floating=False, verbose=2, scoring='accuracy', fixed_features=(0, 2), cv=3) sfs1 = sfs1.fit(X, y) [Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers. [Parallel(n_jobs=1)]: Done 1 out of 1 | elapsed: 0.0s remaining: 0.0s [Parallel(n_jobs=1)]: Done 2 out of 2 | elapsed: 0.0s finished [2023-05-17 08:39:57] Features: 3/4 -- score: 0.9733333333333333[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers. [Parallel(n_jobs=1)]: Done 1 out of 1 | elapsed: 0.0s remaining: 0.0s [Parallel(n_jobs=1)]: Done 1 out of 1 | elapsed: 0.0s finished [2023-05-17 08:39:57] Features: 4/4 -- score: 0.9733333333333333 sfs1.subsets_ {2: {'feature_idx': (0, 2), 'cv_scores': array([0.98, 0.92, 0.94]), 'avg_score': 0.9466666666666667, 'feature_names': ('0', '2')}, 3: {'feature_idx': (0, 2, 3), 'cv_scores': array([0.98, 0.96, 0.98]), 'avg_score': 0.9733333333333333, 'feature_names': ('0', '2', '3')}, 4: {'feature_idx': (0, 1, 2, 3), 'cv_scores': array([0.98, 0.96, 0.98]), 'avg_score': 0.9733333333333333, 'feature_names': ('0', '1', '2', '3')}} If the input dataset is a pandas DataFrame, we can also use the column names directly: import pandas as pd X_df = pd.DataFrame(X, columns=['sepal len', 'petal len', 'sepal width', 'petal width']) X_df.head() .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } sepal len petal len sepal width petal width 0 5.1 3.5 1.4 0.2 1 4.9 3.0 1.4 0.2 2 4.7 3.2 1.3 0.2 3 4.6 3.1 1.5 0.2 4 5.0 3.6 1.4 0.2 sfs2 = SFS(knn, k_features=4, forward=True, floating=False, verbose=2, scoring='accuracy', fixed_features=('sepal len', 'petal len'), cv=3) sfs2 = sfs2.fit(X_df, y_series) [Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers. [Parallel(n_jobs=1)]: Done 1 out of 1 | elapsed: 0.0s remaining: 0.0s [Parallel(n_jobs=1)]: Done 2 out of 2 | elapsed: 0.0s finished [2023-05-17 08:39:57] Features: 3/4 -- score: 0.9466666666666667[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers. [Parallel(n_jobs=1)]: Done 1 out of 1 | elapsed: 0.0s remaining: 0.0s [Parallel(n_jobs=1)]: Done 1 out of 1 | elapsed: 0.0s finished [2023-05-17 08:39:57] Features: 4/4 -- score: 0.9733333333333333 sfs2.subsets_ {2: {'feature_idx': (0, 1), 'cv_scores': array([0.72, 0.74, 0.78]), 'avg_score': 0.7466666666666667, 'feature_names': ('sepal len', 'petal len')}, 3: {'feature_idx': (0, 1, 2), 'cv_scores': array([0.98, 0.92, 0.94]), 'avg_score': 0.9466666666666667, 'feature_names': ('sepal len', 'petal len', 'sepal width')}, 4: {'feature_idx': (0, 1, 2, 3), 'cv_scores': array([0.98, 0.96, 0.98]), 'avg_score': 0.9733333333333333, 'feature_names': ('sepal len', 'petal len', 'sepal width', 'petal width')}}","title":"Example 13 - Specifying Fixed Feature Sets"},{"location":"user_guide/feature_selection/SequentialFeatureSelector/#example-13-working-with-feature-groups","text":"Since mlxtend v0.21.0, it is possible to specify feature groups. Feature groups allow you to group certain features together, such that they are always selected as a group. This can be very useful in contexts similar to one-hot encoding -- if you want to treat the one-hot encoded feature as a single feature: In the following example, we specify sepal length and sepal width as a feature group so that they are always selected together: from sklearn.datasets import load_iris import pandas as pd iris = load_iris() X = iris.data y = iris.target X_df = pd.DataFrame(X, columns=['sepal len', 'petal len', 'sepal wid', 'petal wid']) X_df.head() .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } sepal len petal len sepal wid petal wid 0 5.1 3.5 1.4 0.2 1 4.9 3.0 1.4 0.2 2 4.7 3.2 1.3 0.2 3 4.6 3.1 1.5 0.2 4 5.0 3.6 1.4 0.2 from sklearn.neighbors import KNeighborsClassifier from mlxtend.feature_selection import SequentialFeatureSelector as SFS knn = KNeighborsClassifier(n_neighbors=3) sfs1 = SFS(knn, k_features=2, scoring='accuracy', feature_groups=(['sepal len', 'sepal wid'], ['petal len'], ['petal wid']), cv=3) sfs1 = sfs1.fit(X_df, y) sfs1 = SFS(knn, k_features=2, scoring='accuracy', feature_groups=[[0, 2], [1], [3]], cv=3) sfs1 = sfs1.fit(X, y)","title":"Example 13 - Working with Feature Groups"},{"location":"user_guide/feature_selection/SequentialFeatureSelector/#example-14-multiclass-metrics","text":"Certain scoring metrics like ROC AUC are originally designed for binary classification. However, they can also be used for multiclass settings. It is best to consult this scikit-learn metrics table for this. For example, we can use a ROC AUC One-Vs-Rest score via \u2018\"roc_auc_ovr\" as shown below. from sklearn.datasets import make_blobs X, y = make_blobs(n_samples=10, centers=4, n_features=5, random_state=0) from mlxtend.feature_selection import SequentialFeatureSelector as SFS sfs1 = SFS(knn, k_features=3, forward=True, floating=False, verbose=2, scoring='roc_auc_ovr', cv=0) sfs1 = sfs1.fit(X, y) [Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers. [Parallel(n_jobs=1)]: Done 1 out of 1 | elapsed: 0.0s remaining: 0.0s [Parallel(n_jobs=1)]: Done 5 out of 5 | elapsed: 0.0s finished [2023-05-17 08:39:57] Features: 1/3 -- score: 1.0[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers. [Parallel(n_jobs=1)]: Done 1 out of 1 | elapsed: 0.0s remaining: 0.0s [Parallel(n_jobs=1)]: Done 4 out of 4 | elapsed: 0.0s finished [2023-05-17 08:39:57] Features: 2/3 -- score: 1.0[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers. [Parallel(n_jobs=1)]: Done 1 out of 1 | elapsed: 0.0s remaining: 0.0s [Parallel(n_jobs=1)]: Done 3 out of 3 | elapsed: 0.0s finished [2023-05-17 08:39:57] Features: 3/3 -- score: 1.0","title":"Example 14 - Multiclass Metrics"},{"location":"user_guide/feature_selection/SequentialFeatureSelector/#api","text":"SequentialFeatureSelector(estimator, k_features=1, forward=True, floating=False, verbose=0, scoring=None, cv=5, n_jobs=1, pre_dispatch='2 n_jobs', clone_estimator=True, fixed_features=None, feature_groups=None)* Sequential Feature Selection for Classification and Regression. Parameters estimator : scikit-learn classifier or regressor k_features : int or tuple or str (default: 1) Number of features to select, where k_features < the full feature set. New in 0.4.2: A tuple containing a min and max value can be provided, and the SFS will consider return any feature combination between min and max that scored highest in cross-validation. For example, the tuple (1, 4) will return any combination from 1 up to 4 features instead of a fixed number of features k. New in 0.8.0: A string argument \"best\" or \"parsimonious\". If \"best\" is provided, the feature selector will return the feature subset with the best cross-validation performance. If \"parsimonious\" is provided as an argument, the smallest feature subset that is within one standard error of the cross-validation performance will be selected. forward : bool (default: True) Forward selection if True, backward selection otherwise floating : bool (default: False) Adds a conditional exclusion/inclusion if True. verbose : int (default: 0), level of verbosity to use in logging. If 0, no output, if 1 number of features in current set, if 2 detailed logging i ncluding timestamp and cv scores at step. scoring : str, callable, or None (default: None) If None (default), uses 'accuracy' for sklearn classifiers and 'r2' for sklearn regressors. If str, uses a sklearn scoring metric string identifier, for example {accuracy, f1, precision, recall, roc_auc} for classifiers, {'mean_absolute_error', 'mean_squared_error'/'neg_mean_squared_error', 'median_absolute_error', 'r2'} for regressors. If a callable object or function is provided, it has to be conform with sklearn's signature scorer(estimator, X, y) ; see https://scikit-learn.org/stable/modules/generated/sklearn.metrics.make_scorer.html for more information. cv : int (default: 5) Integer or iterable yielding train, test splits. If cv is an integer and estimator is a classifier (or y consists of integer class labels) stratified k-fold. Otherwise regular k-fold cross-validation is performed. No cross-validation if cv is None, False, or 0. n_jobs : int (default: 1) The number of CPUs to use for evaluating different feature subsets in parallel. -1 means 'all CPUs'. pre_dispatch : int, or string (default: '2*n_jobs') Controls the number of jobs that get dispatched during parallel execution if n_jobs > 1 or n_jobs=-1 . Reducing this number can be useful to avoid an explosion of memory consumption when more jobs get dispatched than CPUs can process. This parameter can be: None, in which case all the jobs are immediately created and spawned. Use this for lightweight and fast-running jobs, to avoid delays due to on-demand spawning of the jobs An int, giving the exact number of total jobs that are spawned A string, giving an expression as a function of n_jobs, as in 2*n_jobs clone_estimator : bool (default: True) Clones estimator if True; works with the original estimator instance if False. Set to False if the estimator doesn't implement scikit-learn's set_params and get_params methods. In addition, it is required to set cv=0, and n_jobs=1. fixed_features : tuple (default: None) If not None , the feature indices provided as a tuple will be regarded as fixed by the feature selector. For example, if fixed_features=(1, 3, 7) , the 2nd, 4th, and 8th feature are guaranteed to be present in the solution. Note that if fixed_features is not None , make sure that the number of features to be selected is greater than len(fixed_features) . In other words, ensure that k_features > len(fixed_features) . New in mlxtend v. 0.18.0. feature_groups : list or None (default: None) Optional argument for treating certain features as a group. This means, the features within a group are always selected together, never split. For example, feature_groups=[[1], [2], [3, 4, 5]] specifies 3 feature groups. In this case, possible feature selection results with k_features=2 are [[1], [2] , [[1], [3, 4, 5]] , or [[2], [3, 4, 5]] . Feature groups can be useful for interpretability, for example, if features 3, 4, 5 are one-hot encoded features. (For more details, please read the notes at the bottom of this docstring). New in mlxtend v. 0.21.0. Attributes k_feature_idx_ : array-like, shape = [n_predictions] Feature Indices of the selected feature subsets. k_feature_names_ : array-like, shape = [n_predictions] Feature names of the selected feature subsets. If pandas DataFrames are used in the fit method, the feature names correspond to the column names. Otherwise, the feature names are string representation of the feature array indices. New in v 0.13.0. k_score_ : float Cross validation average score of the selected subset. subsets_ : dict A dictionary of selected feature subsets during the sequential selection, where the dictionary keys are the lengths k of these feature subsets. If the parameter feature_groups is not None, the value of key indicates the number of groups that are selected together. The dictionary values are dictionaries themselves with the following keys: 'feature_idx' (tuple of indices of the feature subset) 'feature_names' (tuple of feature names of the feat. subset) 'cv_scores' (list individual cross-validation scores) 'avg_score' (average cross-validation score) Note that if pandas DataFrames are used in the fit method, the 'feature_names' correspond to the column names. Otherwise, the feature names are string representation of the feature array indices. The 'feature_names' is new in v 0.13.0. Notes (1) If parameter feature_groups is not None, the number of features is equal to the number of feature groups, i.e. len(feature_groups) . For example, if feature_groups = [[0], [1], [2, 3], [4]] , then the max_features value cannot exceed 4. (2) Although two or more individual features may be considered as one group throughout the feature-selection process, it does not mean the individual features of that group have the same impact on the outcome. For instance, in linear regression, the coefficient of the feature 2 and 3 can be different even if they are considered as one group in feature_groups. (3) If both fixed_features and feature_groups are specified, ensure that each feature group contains the fixed_features selection. E.g., for a 3-feature set fixed_features=[0, 1] and feature_groups=[[0, 1], [2]] is valid; fixed_features=[0, 1] and feature_groups=[[0], [1, 2]] is not valid. (4) In case of KeyboardInterrupt, the dictionary subsets may not be completed. If user is still interested in getting the best score, they can use method `finalize_fit`. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/feature_selection/SequentialFeatureSelector/","title":"API"},{"location":"user_guide/feature_selection/SequentialFeatureSelector/#methods","text":"finalize_fit() None fit(X, y, groups=None, fit_params) Perform feature selection and learn model from training data. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. New in v 0.13.0: pandas DataFrames are now also accepted as argument for X. y : array-like, shape = [n_samples] Target values. New in v 0.13.0: pandas DataFrames are now also accepted as argument for y. groups : array-like, with shape (n_samples,), optional Group labels for the samples used while splitting the dataset into train/test set. Passed to the fit method of the cross-validator. fit_params : various, optional Additional parameters that are being passed to the estimator. For example, sample_weights=weights . Returns self : object fit_transform(X, y, groups=None, fit_params) Fit to training data then reduce X to its most important features. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. New in v 0.13.0: pandas DataFrames are now also accepted as argument for X. y : array-like, shape = [n_samples] Target values. New in v 0.13.0: a pandas Series are now also accepted as argument for y. groups : array-like, with shape (n_samples,), optional Group labels for the samples used while splitting the dataset into train/test set. Passed to the fit method of the cross-validator. fit_params : various, optional Additional parameters that are being passed to the estimator. For example, sample_weights=weights . Returns Reduced feature subset of X, shape={n_samples, k_features} generate_error_message_k_features(name) None get_metric_dict(confidence_interval=0.95) Return metric dictionary Parameters confidence_interval : float (default: 0.95) A positive float between 0.0 and 1.0 to compute the confidence interval bounds of the CV score averages. Returns Dictionary with items where each dictionary value is a list with the number of iterations (number of feature subsets) as its length. The dictionary keys corresponding to these lists are as follows: 'feature_idx': tuple of the indices of the feature subset 'cv_scores': list with individual CV scores 'avg_score': of CV average scores 'std_dev': standard deviation of the CV score average 'std_err': standard error of the CV score average 'ci_bound': confidence interval bound of the CV score average get_params(deep=True) Get parameters for this estimator. Parameters deep : bool, default=True If True, will return the parameters for this estimator and contained subobjects that are estimators. Returns params : dict Parameter names mapped to their values. set_params( params) Set the parameters of this estimator. Valid parameter keys can be listed with get_params() . Returns self transform(X) Reduce X to its most important features. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. New in v 0.13.0: pandas DataFrames are now also accepted as argument for X. Returns Reduced feature subset of X, shape={n_samples, k_features}","title":"Methods"},{"location":"user_guide/feature_selection/SequentialFeatureSelector/#properties","text":"named_estimators Returns List of named estimator tuples, like [('svc', SVC(...))] ython","title":"Properties"},{"location":"user_guide/file_io/find_filegroups/","text":"find_filegroups: Find files that only differ via their file extensions A function that finds files that belong together (i.e., differ only by file extension) in different directories and collects them in a Python dictionary for further processing tasks. from mlxtend.file_io import find_filegroups Overview This function finds files that are related to each other based on their file names. This can be useful for parsing collections files that have been stored in different subdirectories, for examples: input_dir/ task01.txt task02.txt ... log_dir/ task01.log task02.log ... output_dir/ task01.dat task02.dat ... References - Example 1 - Grouping related files in a dictionary Given the following directory and file structure dir_1/ file_1.log file_2.log file_3.log dir_2/ file_1.csv file_2.csv file_3.csv dir_3/ file_1.txt file_2.txt file_3.txt we can use find_filegroups to group related files as items of a dictionary as shown below: from mlxtend.file_io import find_filegroups find_filegroups(paths=['./data_find_filegroups/dir_1', './data_find_filegroups/dir_2', './data_find_filegroups/dir_3'], substring='file_') {'file_1': ['./data_find_filegroups/dir_1/file_1.log', './data_find_filegroups/dir_2/file_1.csv', './data_find_filegroups/dir_3/file_1.txt'], 'file_2': ['./data_find_filegroups/dir_1/file_2.log', './data_find_filegroups/dir_2/file_2.csv', './data_find_filegroups/dir_3/file_2.txt'], 'file_3': ['./data_find_filegroups/dir_1/file_3.log', './data_find_filegroups/dir_2/file_3.csv', './data_find_filegroups/dir_3/file_3.txt']} API find_filegroups(paths, substring='', extensions=None, validity_check=True, ignore_invisible=True, rstrip='', ignore_substring=None) Find and collect files from different directories in a python dictionary. Parameters paths : list Paths of the directories to be searched. Dictionary keys are build from the first directory. substring : str (default: '') Substring that all files have to contain to be considered. extensions : list (default: None) None or list of allowed file extensions for each path. If provided, the number of extensions must match the number of paths . validity_check : bool (default: None) If True , checks if all dictionary values have the same number of file paths. Prints a warning and returns an empty dictionary if the validity check failed. ignore_invisible : bool (default: True) If True , ignores invisible files (i.e., files starting with a period). rstrip : str (default: '') If provided, strips characters from right side of the file base names after splitting the extension. Useful to trim different filenames to a common stem. E.g,. \"abc_d.txt\" and \"abc_d_.csv\" would share the stem \"abc_d\" if rstrip is set to \"_\". ignore_substring : str (default: None) Ignores files that contain the specified substring. Returns groups : dict Dictionary of files paths. Keys are the file names found in the first directory listed in paths (without file extension). Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/file_io/find_filegroups/","title":"find_filegroups: Find files that only differ via their file extensions"},{"location":"user_guide/file_io/find_filegroups/#find_filegroups-find-files-that-only-differ-via-their-file-extensions","text":"A function that finds files that belong together (i.e., differ only by file extension) in different directories and collects them in a Python dictionary for further processing tasks. from mlxtend.file_io import find_filegroups","title":"find_filegroups: Find files that only differ via their file extensions"},{"location":"user_guide/file_io/find_filegroups/#overview","text":"This function finds files that are related to each other based on their file names. This can be useful for parsing collections files that have been stored in different subdirectories, for examples: input_dir/ task01.txt task02.txt ... log_dir/ task01.log task02.log ... output_dir/ task01.dat task02.dat ...","title":"Overview"},{"location":"user_guide/file_io/find_filegroups/#references","text":"-","title":"References"},{"location":"user_guide/file_io/find_filegroups/#example-1-grouping-related-files-in-a-dictionary","text":"Given the following directory and file structure dir_1/ file_1.log file_2.log file_3.log dir_2/ file_1.csv file_2.csv file_3.csv dir_3/ file_1.txt file_2.txt file_3.txt we can use find_filegroups to group related files as items of a dictionary as shown below: from mlxtend.file_io import find_filegroups find_filegroups(paths=['./data_find_filegroups/dir_1', './data_find_filegroups/dir_2', './data_find_filegroups/dir_3'], substring='file_') {'file_1': ['./data_find_filegroups/dir_1/file_1.log', './data_find_filegroups/dir_2/file_1.csv', './data_find_filegroups/dir_3/file_1.txt'], 'file_2': ['./data_find_filegroups/dir_1/file_2.log', './data_find_filegroups/dir_2/file_2.csv', './data_find_filegroups/dir_3/file_2.txt'], 'file_3': ['./data_find_filegroups/dir_1/file_3.log', './data_find_filegroups/dir_2/file_3.csv', './data_find_filegroups/dir_3/file_3.txt']}","title":"Example 1 - Grouping related files in a dictionary"},{"location":"user_guide/file_io/find_filegroups/#api","text":"find_filegroups(paths, substring='', extensions=None, validity_check=True, ignore_invisible=True, rstrip='', ignore_substring=None) Find and collect files from different directories in a python dictionary. Parameters paths : list Paths of the directories to be searched. Dictionary keys are build from the first directory. substring : str (default: '') Substring that all files have to contain to be considered. extensions : list (default: None) None or list of allowed file extensions for each path. If provided, the number of extensions must match the number of paths . validity_check : bool (default: None) If True , checks if all dictionary values have the same number of file paths. Prints a warning and returns an empty dictionary if the validity check failed. ignore_invisible : bool (default: True) If True , ignores invisible files (i.e., files starting with a period). rstrip : str (default: '') If provided, strips characters from right side of the file base names after splitting the extension. Useful to trim different filenames to a common stem. E.g,. \"abc_d.txt\" and \"abc_d_.csv\" would share the stem \"abc_d\" if rstrip is set to \"_\". ignore_substring : str (default: None) Ignores files that contain the specified substring. Returns groups : dict Dictionary of files paths. Keys are the file names found in the first directory listed in paths (without file extension). Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/file_io/find_filegroups/","title":"API"},{"location":"user_guide/file_io/find_files/","text":"find_files: Find files based on substring matches A function that finds files in a given directory based on substring matches and returns a list of the file names found. from mlxtend.file_io import find_files Overview This function finds files based on substring search. This is especially useful if we want to find specific files in a directory tree and return their absolute paths for further processing in Python. References - Example 1 - Grouping related files in a dictionary Given the following directory and file structure dir_1/ file_1.log file_2.log file_3.log dir_2/ file_1.csv file_2.csv file_3.csv dir_3/ file_1.txt file_2.txt file_3.txt we can use find_files to return the paths to all files that contain the substring _2 as follows: from mlxtend.file_io import find_files find_files(substring='_2', path='./data_find_filegroups/', recursive=True) ['./data_find_filegroups/dir_1/file_2.log', './data_find_filegroups/dir_2/file_2.csv', './data_find_filegroups/dir_3/file_2.txt'] API find_files(substring, path, recursive=False, check_ext=None, ignore_invisible=True, ignore_substring=None) Find files in a directory based on substring matching. Parameters substring : str Substring of the file to be matched. path : str Path where to look. recursive : bool If true, searches subdirectories recursively. check_ext : str If string (e.g., '.txt'), only returns files that match the specified file extension. ignore_invisible : bool If True , ignores invisible files (i.e., files starting with a period). ignore_substring : str Ignores files that contain the specified substring. Returns results : list List of the matched files. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/file_io/find_files/","title":"find_files: Find files based on substring matches"},{"location":"user_guide/file_io/find_files/#find_files-find-files-based-on-substring-matches","text":"A function that finds files in a given directory based on substring matches and returns a list of the file names found. from mlxtend.file_io import find_files","title":"find_files: Find files based on substring matches"},{"location":"user_guide/file_io/find_files/#overview","text":"This function finds files based on substring search. This is especially useful if we want to find specific files in a directory tree and return their absolute paths for further processing in Python.","title":"Overview"},{"location":"user_guide/file_io/find_files/#references","text":"-","title":"References"},{"location":"user_guide/file_io/find_files/#example-1-grouping-related-files-in-a-dictionary","text":"Given the following directory and file structure dir_1/ file_1.log file_2.log file_3.log dir_2/ file_1.csv file_2.csv file_3.csv dir_3/ file_1.txt file_2.txt file_3.txt we can use find_files to return the paths to all files that contain the substring _2 as follows: from mlxtend.file_io import find_files find_files(substring='_2', path='./data_find_filegroups/', recursive=True) ['./data_find_filegroups/dir_1/file_2.log', './data_find_filegroups/dir_2/file_2.csv', './data_find_filegroups/dir_3/file_2.txt']","title":"Example 1 - Grouping related files in a dictionary"},{"location":"user_guide/file_io/find_files/#api","text":"find_files(substring, path, recursive=False, check_ext=None, ignore_invisible=True, ignore_substring=None) Find files in a directory based on substring matching. Parameters substring : str Substring of the file to be matched. path : str Path where to look. recursive : bool If true, searches subdirectories recursively. check_ext : str If string (e.g., '.txt'), only returns files that match the specified file extension. ignore_invisible : bool If True , ignores invisible files (i.e., files starting with a period). ignore_substring : str Ignores files that contain the specified substring. Returns results : list List of the matched files. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/file_io/find_files/","title":"API"},{"location":"user_guide/frequent_patterns/apriori/","text":"apriori: Frequent itemsets via the Apriori algorithm Apriori function to extract frequent itemsets for association rule mining from mlxtend.frequent_patterns import apriori Overview Apriori is a popular algorithm [1] for extracting frequent itemsets with applications in association rule learning. The apriori algorithm has been designed to operate on databases containing transactions, such as purchases by customers of a store. An itemset is considered as \"frequent\" if it meets a user-specified support threshold. For instance, if the support threshold is set to 0.5 (50%), a frequent itemset is defined as a set of items that occur together in at least 50% of all transactions in the database. References [1] Agrawal, Rakesh, and Ramakrishnan Srikant. \" Fast algorithms for mining association rules .\" Proc. 20th int. conf. very large data bases, VLDB. Vol. 1215. 1994. Related FP-Growth FP-Max Example 1 -- Generating Frequent Itemsets The apriori function expects data in a one-hot encoded pandas DataFrame. Suppose we have the following transaction data: dataset = [['Milk', 'Onion', 'Nutmeg', 'Kidney Beans', 'Eggs', 'Yogurt'], ['Dill', 'Onion', 'Nutmeg', 'Kidney Beans', 'Eggs', 'Yogurt'], ['Milk', 'Apple', 'Kidney Beans', 'Eggs'], ['Milk', 'Unicorn', 'Corn', 'Kidney Beans', 'Yogurt'], ['Corn', 'Onion', 'Onion', 'Kidney Beans', 'Ice cream', 'Eggs']] We can transform it into the right format via the TransactionEncoder as follows: import pandas as pd from mlxtend.preprocessing import TransactionEncoder te = TransactionEncoder() te_ary = te.fit(dataset).transform(dataset) df = pd.DataFrame(te_ary, columns=te.columns_) df .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } Apple Corn Dill Eggs Ice cream Kidney Beans Milk Nutmeg Onion Unicorn Yogurt 0 False False False True False True True True True False True 1 False False True True False True False True True False True 2 True False False True False True True False False False False 3 False True False False False True True False False True True 4 False True False True True True False False True False False Now, let us return the items and itemsets with at least 60% support: from mlxtend.frequent_patterns import apriori apriori(df, min_support=0.6) .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } support itemsets 0 0.8 (3) 1 1.0 (5) 2 0.6 (6) 3 0.6 (8) 4 0.6 (10) 5 0.8 (3, 5) 6 0.6 (8, 3) 7 0.6 (5, 6) 8 0.6 (8, 5) 9 0.6 (10, 5) 10 0.6 (8, 3, 5) By default, apriori returns the column indices of the items, which may be useful in downstream operations such as association rule mining. For better readability, we can set use_colnames=True to convert these integer values into the respective item names: apriori(df, min_support=0.6, use_colnames=True) .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } support itemsets 0 0.8 (Eggs) 1 1.0 (Kidney Beans) 2 0.6 (Milk) 3 0.6 (Onion) 4 0.6 (Yogurt) 5 0.8 (Eggs, Kidney Beans) 6 0.6 (Eggs, Onion) 7 0.6 (Kidney Beans, Milk) 8 0.6 (Kidney Beans, Onion) 9 0.6 (Yogurt, Kidney Beans) 10 0.6 (Kidney Beans, Eggs, Onion) Example 2 -- Selecting and Filtering Results The advantage of working with pandas DataFrames is that we can use its convenient features to filter the results. For instance, let's assume we are only interested in itemsets of length 2 that have a support of at least 80 percent. First, we create the frequent itemsets via apriori and add a new column that stores the length of each itemset: frequent_itemsets = apriori(df, min_support=0.6, use_colnames=True) frequent_itemsets['length'] = frequent_itemsets['itemsets'].apply(lambda x: len(x)) frequent_itemsets .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } support itemsets length 0 0.8 (Eggs) 1 1 1.0 (Kidney Beans) 1 2 0.6 (Milk) 1 3 0.6 (Onion) 1 4 0.6 (Yogurt) 1 5 0.8 (Eggs, Kidney Beans) 2 6 0.6 (Eggs, Onion) 2 7 0.6 (Kidney Beans, Milk) 2 8 0.6 (Kidney Beans, Onion) 2 9 0.6 (Yogurt, Kidney Beans) 2 10 0.6 (Kidney Beans, Eggs, Onion) 3 Then, we can select the results that satisfy our desired criteria as follows: frequent_itemsets[ (frequent_itemsets['length'] == 2) & (frequent_itemsets['support'] >= 0.8) ] .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } support itemsets length 5 0.8 (Eggs, Kidney Beans) 2 Similarly, using the Pandas API, we can select entries based on the \"itemsets\" column: frequent_itemsets[ frequent_itemsets['itemsets'] == {'Onion', 'Eggs'} ] .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } support itemsets length 6 0.6 (Eggs, Onion) 2 Frozensets Note that the entries in the \"itemsets\" column are of type frozenset , which is built-in Python type that is similar to a Python set but immutable, which makes it more efficient for certain query or comparison operations (https://docs.python.org/3.6/library/stdtypes.html#frozenset). Since frozenset s are sets, the item order does not matter. I.e., the query frequent_itemsets[ frequent_itemsets['itemsets'] == {'Onion', 'Eggs'} ] is equivalent to any of the following three frequent_itemsets[ frequent_itemsets['itemsets'] == {'Eggs', 'Onion'} ] frequent_itemsets[ frequent_itemsets['itemsets'] == frozenset(('Eggs', 'Onion')) ] frequent_itemsets[ frequent_itemsets['itemsets'] == frozenset(('Onion', 'Eggs')) ] Example 3 -- Working with Sparse Representations To save memory, you may want to represent your transaction data in the sparse format. This is especially useful if you have lots of products and small transactions. oht_ary = te.fit(dataset).transform(dataset, sparse=True) sparse_df = pd.DataFrame.sparse.from_spmatrix(oht_ary, columns=te.columns_) sparse_df .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } Apple Corn Dill Eggs Ice cream Kidney Beans Milk Nutmeg Onion Unicorn Yogurt 0 False False False True False True True True True False True 1 False False True True False True False True True False True 2 True False False True False True True False False False False 3 False True False False False True True False False True True 4 False True False True True True False False True False False apriori(sparse_df, min_support=0.6, use_colnames=True, verbose=1) Processing 21 combinations | Sampling itemset size 3 .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } support itemsets 0 0.8 (Eggs) 1 1.0 (Kidney Beans) 2 0.6 (Milk) 3 0.6 (Onion) 4 0.6 (Yogurt) 5 0.8 (Eggs, Kidney Beans) 6 0.6 (Eggs, Onion) 7 0.6 (Kidney Beans, Milk) 8 0.6 (Kidney Beans, Onion) 9 0.6 (Yogurt, Kidney Beans) 10 0.6 (Kidney Beans, Eggs, Onion) API apriori(df, min_support=0.5, use_colnames=False, max_len=None, verbose=0, low_memory=False) Get frequent itemsets from a one-hot DataFrame Parameters df : pandas DataFrame pandas DataFrame the encoded format. Also supports DataFrames with sparse data; for more info, please see (https://pandas.pydata.org/pandas-docs/stable/ user_guide/sparse.html#sparse-data-structures) Please note that the old pandas SparseDataFrame format is no longer supported in mlxtend >= 0.17.2. The allowed values are either 0/1 or True/False. For example, Apple Bananas Beer Chicken Milk Rice 0 True False True True False True 1 True False True False False True 2 True False True False False False 3 True True False False False False 4 False False True True True True 5 False False True False True True 6 False False True False True False 7 True True False False False False min_support : float (default: 0.5) A float between 0 and 1 for minumum support of the itemsets returned. The support is computed as the fraction transactions_where_item(s)_occur / total_transactions . use_colnames : bool (default: False) If True , uses the DataFrames' column names in the returned DataFrame instead of column indices. max_len : int (default: None) Maximum length of the itemsets generated. If None (default) all possible itemsets lengths (under the apriori condition) are evaluated. verbose : int (default: 0) Shows the number of iterations if >= 1 and low_memory is True . If =1 and low_memory is False , shows the number of combinations. low_memory : bool (default: False) If True , uses an iterator to search for combinations above min_support . Note that while low_memory=True should only be used for large dataset if memory resources are limited, because this implementation is approx. 3-6x slower than the default. Returns pandas DataFrame with columns ['support', 'itemsets'] of all itemsets that are >= min_support and < than max_len (if max_len is not None). Each itemset in the 'itemsets' column is of type frozenset , which is a Python built-in type that behaves similarly to sets except that it is immutable (For more info, see https://docs.python.org/3.6/library/stdtypes.html#frozenset). Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/frequent_patterns/apriori/","title":"Apriori"},{"location":"user_guide/frequent_patterns/apriori/#apriori-frequent-itemsets-via-the-apriori-algorithm","text":"Apriori function to extract frequent itemsets for association rule mining from mlxtend.frequent_patterns import apriori","title":"apriori: Frequent itemsets via the Apriori algorithm"},{"location":"user_guide/frequent_patterns/apriori/#overview","text":"Apriori is a popular algorithm [1] for extracting frequent itemsets with applications in association rule learning. The apriori algorithm has been designed to operate on databases containing transactions, such as purchases by customers of a store. An itemset is considered as \"frequent\" if it meets a user-specified support threshold. For instance, if the support threshold is set to 0.5 (50%), a frequent itemset is defined as a set of items that occur together in at least 50% of all transactions in the database.","title":"Overview"},{"location":"user_guide/frequent_patterns/apriori/#references","text":"[1] Agrawal, Rakesh, and Ramakrishnan Srikant. \" Fast algorithms for mining association rules .\" Proc. 20th int. conf. very large data bases, VLDB. Vol. 1215. 1994.","title":"References"},{"location":"user_guide/frequent_patterns/apriori/#related","text":"FP-Growth FP-Max","title":"Related"},{"location":"user_guide/frequent_patterns/apriori/#example-1-generating-frequent-itemsets","text":"The apriori function expects data in a one-hot encoded pandas DataFrame. Suppose we have the following transaction data: dataset = [['Milk', 'Onion', 'Nutmeg', 'Kidney Beans', 'Eggs', 'Yogurt'], ['Dill', 'Onion', 'Nutmeg', 'Kidney Beans', 'Eggs', 'Yogurt'], ['Milk', 'Apple', 'Kidney Beans', 'Eggs'], ['Milk', 'Unicorn', 'Corn', 'Kidney Beans', 'Yogurt'], ['Corn', 'Onion', 'Onion', 'Kidney Beans', 'Ice cream', 'Eggs']] We can transform it into the right format via the TransactionEncoder as follows: import pandas as pd from mlxtend.preprocessing import TransactionEncoder te = TransactionEncoder() te_ary = te.fit(dataset).transform(dataset) df = pd.DataFrame(te_ary, columns=te.columns_) df .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } Apple Corn Dill Eggs Ice cream Kidney Beans Milk Nutmeg Onion Unicorn Yogurt 0 False False False True False True True True True False True 1 False False True True False True False True True False True 2 True False False True False True True False False False False 3 False True False False False True True False False True True 4 False True False True True True False False True False False Now, let us return the items and itemsets with at least 60% support: from mlxtend.frequent_patterns import apriori apriori(df, min_support=0.6) .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } support itemsets 0 0.8 (3) 1 1.0 (5) 2 0.6 (6) 3 0.6 (8) 4 0.6 (10) 5 0.8 (3, 5) 6 0.6 (8, 3) 7 0.6 (5, 6) 8 0.6 (8, 5) 9 0.6 (10, 5) 10 0.6 (8, 3, 5) By default, apriori returns the column indices of the items, which may be useful in downstream operations such as association rule mining. For better readability, we can set use_colnames=True to convert these integer values into the respective item names: apriori(df, min_support=0.6, use_colnames=True) .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } support itemsets 0 0.8 (Eggs) 1 1.0 (Kidney Beans) 2 0.6 (Milk) 3 0.6 (Onion) 4 0.6 (Yogurt) 5 0.8 (Eggs, Kidney Beans) 6 0.6 (Eggs, Onion) 7 0.6 (Kidney Beans, Milk) 8 0.6 (Kidney Beans, Onion) 9 0.6 (Yogurt, Kidney Beans) 10 0.6 (Kidney Beans, Eggs, Onion)","title":"Example 1 -- Generating Frequent Itemsets"},{"location":"user_guide/frequent_patterns/apriori/#example-2-selecting-and-filtering-results","text":"The advantage of working with pandas DataFrames is that we can use its convenient features to filter the results. For instance, let's assume we are only interested in itemsets of length 2 that have a support of at least 80 percent. First, we create the frequent itemsets via apriori and add a new column that stores the length of each itemset: frequent_itemsets = apriori(df, min_support=0.6, use_colnames=True) frequent_itemsets['length'] = frequent_itemsets['itemsets'].apply(lambda x: len(x)) frequent_itemsets .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } support itemsets length 0 0.8 (Eggs) 1 1 1.0 (Kidney Beans) 1 2 0.6 (Milk) 1 3 0.6 (Onion) 1 4 0.6 (Yogurt) 1 5 0.8 (Eggs, Kidney Beans) 2 6 0.6 (Eggs, Onion) 2 7 0.6 (Kidney Beans, Milk) 2 8 0.6 (Kidney Beans, Onion) 2 9 0.6 (Yogurt, Kidney Beans) 2 10 0.6 (Kidney Beans, Eggs, Onion) 3 Then, we can select the results that satisfy our desired criteria as follows: frequent_itemsets[ (frequent_itemsets['length'] == 2) & (frequent_itemsets['support'] >= 0.8) ] .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } support itemsets length 5 0.8 (Eggs, Kidney Beans) 2 Similarly, using the Pandas API, we can select entries based on the \"itemsets\" column: frequent_itemsets[ frequent_itemsets['itemsets'] == {'Onion', 'Eggs'} ] .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } support itemsets length 6 0.6 (Eggs, Onion) 2 Frozensets Note that the entries in the \"itemsets\" column are of type frozenset , which is built-in Python type that is similar to a Python set but immutable, which makes it more efficient for certain query or comparison operations (https://docs.python.org/3.6/library/stdtypes.html#frozenset). Since frozenset s are sets, the item order does not matter. I.e., the query frequent_itemsets[ frequent_itemsets['itemsets'] == {'Onion', 'Eggs'} ] is equivalent to any of the following three frequent_itemsets[ frequent_itemsets['itemsets'] == {'Eggs', 'Onion'} ] frequent_itemsets[ frequent_itemsets['itemsets'] == frozenset(('Eggs', 'Onion')) ] frequent_itemsets[ frequent_itemsets['itemsets'] == frozenset(('Onion', 'Eggs')) ]","title":"Example 2 -- Selecting and Filtering Results"},{"location":"user_guide/frequent_patterns/apriori/#example-3-working-with-sparse-representations","text":"To save memory, you may want to represent your transaction data in the sparse format. This is especially useful if you have lots of products and small transactions. oht_ary = te.fit(dataset).transform(dataset, sparse=True) sparse_df = pd.DataFrame.sparse.from_spmatrix(oht_ary, columns=te.columns_) sparse_df .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } Apple Corn Dill Eggs Ice cream Kidney Beans Milk Nutmeg Onion Unicorn Yogurt 0 False False False True False True True True True False True 1 False False True True False True False True True False True 2 True False False True False True True False False False False 3 False True False False False True True False False True True 4 False True False True True True False False True False False apriori(sparse_df, min_support=0.6, use_colnames=True, verbose=1) Processing 21 combinations | Sampling itemset size 3 .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } support itemsets 0 0.8 (Eggs) 1 1.0 (Kidney Beans) 2 0.6 (Milk) 3 0.6 (Onion) 4 0.6 (Yogurt) 5 0.8 (Eggs, Kidney Beans) 6 0.6 (Eggs, Onion) 7 0.6 (Kidney Beans, Milk) 8 0.6 (Kidney Beans, Onion) 9 0.6 (Yogurt, Kidney Beans) 10 0.6 (Kidney Beans, Eggs, Onion)","title":"Example 3 -- Working with Sparse Representations"},{"location":"user_guide/frequent_patterns/apriori/#api","text":"apriori(df, min_support=0.5, use_colnames=False, max_len=None, verbose=0, low_memory=False) Get frequent itemsets from a one-hot DataFrame Parameters df : pandas DataFrame pandas DataFrame the encoded format. Also supports DataFrames with sparse data; for more info, please see (https://pandas.pydata.org/pandas-docs/stable/ user_guide/sparse.html#sparse-data-structures) Please note that the old pandas SparseDataFrame format is no longer supported in mlxtend >= 0.17.2. The allowed values are either 0/1 or True/False. For example, Apple Bananas Beer Chicken Milk Rice 0 True False True True False True 1 True False True False False True 2 True False True False False False 3 True True False False False False 4 False False True True True True 5 False False True False True True 6 False False True False True False 7 True True False False False False min_support : float (default: 0.5) A float between 0 and 1 for minumum support of the itemsets returned. The support is computed as the fraction transactions_where_item(s)_occur / total_transactions . use_colnames : bool (default: False) If True , uses the DataFrames' column names in the returned DataFrame instead of column indices. max_len : int (default: None) Maximum length of the itemsets generated. If None (default) all possible itemsets lengths (under the apriori condition) are evaluated. verbose : int (default: 0) Shows the number of iterations if >= 1 and low_memory is True . If =1 and low_memory is False , shows the number of combinations. low_memory : bool (default: False) If True , uses an iterator to search for combinations above min_support . Note that while low_memory=True should only be used for large dataset if memory resources are limited, because this implementation is approx. 3-6x slower than the default. Returns pandas DataFrame with columns ['support', 'itemsets'] of all itemsets that are >= min_support and < than max_len (if max_len is not None). Each itemset in the 'itemsets' column is of type frozenset , which is a Python built-in type that behaves similarly to sets except that it is immutable (For more info, see https://docs.python.org/3.6/library/stdtypes.html#frozenset). Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/frequent_patterns/apriori/","title":"API"},{"location":"user_guide/frequent_patterns/association_rules/","text":"association_rules: Association rules generation from frequent itemsets Function to generate association rules from frequent itemsets from mlxtend.frequent_patterns import association_rules Overview Rule generation is a common task in the mining of frequent patterns. An association rule is an implication expression of the form X \\rightarrow Y , where X and Y are disjoint itemsets [1]. A more concrete example based on consumer behaviour would be \\{Diapers\\} \\rightarrow \\{Beer\\} suggesting that people who buy diapers are also likely to buy beer. To evaluate the \"interest\" of such an association rule, different metrics have been developed. The current implementation make use of the confidence and lift metrics. Metrics The currently supported metrics for evaluating association rules and setting selection thresholds are listed below. Given a rule \"A -> C\", A stands for antecedent and C stands for consequent. 'support': \\text{support}(A\\rightarrow C) = \\text{support}(A \\cup C), \\;\\;\\; \\text{range: } [0, 1] introduced in [3] The support metric is defined for itemsets, not assocication rules. The table produced by the association rule mining algorithm contains three different support metrics: 'antecedent support', 'consequent support', and 'support'. Here, 'antecedent support' computes the proportion of transactions that contain the antecedent A, and 'consequent support' computes the support for the itemset of the consequent C. The 'support' metric then computes the support of the combined itemset A \\cup C. Typically, support is used to measure the abundance or frequency (often interpreted as significance or importance) of an itemset in a database. We refer to an itemset as a \"frequent itemset\" if you support is larger than a specified minimum-support threshold. Note that in general, due to the downward closure property, all subsets of a frequent itemset are also frequent. 'confidence': \\text{confidence}(A\\rightarrow C) = \\frac{\\text{support}(A\\rightarrow C)}{\\text{support}(A)}, \\;\\;\\; \\text{range: } [0, 1] introduced in [3] The confidence of a rule A->C is the probability of seeing the consequent in a transaction given that it also contains the antecedent. Note that the metric is not symmetric or directed; for instance, the confidence for A->C is different than the confidence for C->A. The confidence is 1 (maximal) for a rule A->C if the consequent and antecedent always occur together. 'lift': \\text{lift}(A\\rightarrow C) = \\frac{\\text{confidence}(A\\rightarrow C)}{\\text{support}(C)}, \\;\\;\\; \\text{range: } [0, \\infty] introduced in [4] The lift metric is commonly used to measure how much more often the antecedent and consequent of a rule A->C occur together than we would expect if they were statistically independent. If A and C are independent, the Lift score will be exactly 1. 'leverage': \\text{levarage}(A\\rightarrow C) = \\text{support}(A\\rightarrow C) - \\text{support}(A) \\times \\text{support}(C), \\;\\;\\; \\text{range: } [-1, 1] introduced in [5] Leverage computes the difference between the observed frequency of A and C appearing together and the frequency that would be expected if A and C were independent. A leverage value of 0 indicates independence. 'conviction': \\text{conviction}(A\\rightarrow C) = \\frac{1 - \\text{support}(C)}{1 - \\text{confidence}(A\\rightarrow C)}, \\;\\;\\; \\text{range: } [0, \\infty] introduced in [6] A high conviction value means that the consequent is highly depending on the antecedent. For instance, in the case of a perfect confidence score, the denominator becomes 0 (due to 1 - 1) for which the conviction score is defined as 'inf'. Similar to lift, if items are independent, the conviction is 1. 'zhangs_metric': \\text{zhangs metric}(A\\rightarrow C) = \\frac{\\text{confidence}(A\\rightarrow C) - \\text{confidence}(A'\\rightarrow C)}{Max[ \\text{confidence}(A\\rightarrow C) , \\text{confidence}(A'\\rightarrow C)]}, \\;\\;\\; \\text{range: } [-1, 1] introduced in [7] Measures both association and dissociation. Value ranges between -1 and 1. A positive value (>0) indicates Association and negative value indicated dissociation. References [1] Tan, Steinbach, Kumar. Introduction to Data Mining. Pearson New International Edition. Harlow: Pearson Education Ltd., 2014. (pp. 327-414). [2] Michael Hahsler, https://michael.hahsler.net/research/association_rules/measures.html [3] R. Agrawal, T. Imielinski, and A. Swami. Mining associations between sets of items in large databases. In Proc. of the ACM SIGMOD Int'l Conference on Management of Data, pages 207-216, Washington D.C., May 1993 [4] S. Brin, R. Motwani, J. D. Ullman, and S. Tsur. Dynamic itemset counting and implication rules for market basket data [5] Piatetsky-Shapiro, G., Discovery, analysis, and presentation of strong rules. Knowledge Discovery in Databases, 1991: p. 229-248. [6] Sergey Brin, Rajeev Motwani, Jeffrey D. Ullman, and Shalom Turk. Dynamic itemset counting and implication rules for market basket data. In SIGMOD 1997, Proceedings ACM SIGMOD International Conference on Management of Data, pages 255-264, Tucson, Arizona, USA, May 1997 [7] Xiaowei Yan , Chengqi Zhang & Shichao Zhang (2009) CONFIDENCE METRICS FOR ASSOCIATION RULE MINING, Applied Artificial Intelligence, 23:8, 713-737 https://www.tandfonline.com/doi/pdf/10.1080/08839510903208062. Example 1 -- Generating Association Rules from Frequent Itemsets The generate_rules takes dataframes of frequent itemsets as produced by the apriori , fpgrowth , or fpmax functions in mlxtend.association . To demonstrate the usage of the generate_rules method, we first create a pandas DataFrame of frequent itemsets as generated by the fpgrowth function: import pandas as pd from mlxtend.preprocessing import TransactionEncoder from mlxtend.frequent_patterns import apriori, fpmax, fpgrowth dataset = [['Milk', 'Onion', 'Nutmeg', 'Kidney Beans', 'Eggs', 'Yogurt'], ['Dill', 'Onion', 'Nutmeg', 'Kidney Beans', 'Eggs', 'Yogurt'], ['Milk', 'Apple', 'Kidney Beans', 'Eggs'], ['Milk', 'Unicorn', 'Corn', 'Kidney Beans', 'Yogurt'], ['Corn', 'Onion', 'Onion', 'Kidney Beans', 'Ice cream', 'Eggs']] te = TransactionEncoder() te_ary = te.fit(dataset).transform(dataset) df = pd.DataFrame(te_ary, columns=te.columns_) frequent_itemsets = fpgrowth(df, min_support=0.6, use_colnames=True) ### alternatively: #frequent_itemsets = apriori(df, min_support=0.6, use_colnames=True) #frequent_itemsets = fpmax(df, min_support=0.6, use_colnames=True) frequent_itemsets .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } support itemsets 0 1.0 (Kidney Beans) 1 0.8 (Eggs) 2 0.6 (Yogurt) 3 0.6 (Onion) 4 0.6 (Milk) 5 0.8 (Kidney Beans, Eggs) 6 0.6 (Kidney Beans, Yogurt) 7 0.6 (Eggs, Onion) 8 0.6 (Kidney Beans, Onion) 9 0.6 (Eggs, Kidney Beans, Onion) 10 0.6 (Kidney Beans, Milk) The generate_rules() function allows you to (1) specify your metric of interest and (2) the according threshold. Currently implemented measures are confidence and lift . Let's say you are interested in rules derived from the frequent itemsets only if the level of confidence is above the 70 percent threshold ( min_threshold=0.7 ): from mlxtend.frequent_patterns import association_rules association_rules(frequent_itemsets, metric=\"confidence\", min_threshold=0.7) .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } antecedents consequents antecedent support consequent support support confidence lift leverage conviction zhangs_metric 0 (Kidney Beans) (Eggs) 1.0 0.8 0.8 0.80 1.00 0.00 1.0 0.0 1 (Eggs) (Kidney Beans) 0.8 1.0 0.8 1.00 1.00 0.00 inf 0.0 2 (Yogurt) (Kidney Beans) 0.6 1.0 0.6 1.00 1.00 0.00 inf 0.0 3 (Eggs) (Onion) 0.8 0.6 0.6 0.75 1.25 0.12 1.6 1.0 4 (Onion) (Eggs) 0.6 0.8 0.6 1.00 1.25 0.12 inf 0.5 5 (Onion) (Kidney Beans) 0.6 1.0 0.6 1.00 1.00 0.00 inf 0.0 6 (Kidney Beans, Eggs) (Onion) 0.8 0.6 0.6 0.75 1.25 0.12 1.6 1.0 7 (Onion, Eggs) (Kidney Beans) 0.6 1.0 0.6 1.00 1.00 0.00 inf 0.0 8 (Kidney Beans, Onion) (Eggs) 0.6 0.8 0.6 1.00 1.25 0.12 inf 0.5 9 (Eggs) (Kidney Beans, Onion) 0.8 0.6 0.6 0.75 1.25 0.12 1.6 1.0 10 (Onion) (Kidney Beans, Eggs) 0.6 0.8 0.6 1.00 1.25 0.12 inf 0.5 11 (Milk) (Kidney Beans) 0.6 1.0 0.6 1.00 1.00 0.00 inf 0.0 Example 2 -- Rule Generation and Selection Criteria If you are interested in rules according to a different metric of interest, you can simply adjust the metric and min_threshold arguments . E.g. if you are only interested in rules that have a lift score of >= 1.2, you would do the following: rules = association_rules(frequent_itemsets, metric=\"lift\", min_threshold=1.2) rules .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } antecedents consequents antecedent support consequent support support confidence lift leverage conviction zhangs_metric 0 (Eggs) (Onion) 0.8 0.6 0.6 0.75 1.25 0.12 1.6 1.0 1 (Onion) (Eggs) 0.6 0.8 0.6 1.00 1.25 0.12 inf 0.5 2 (Kidney Beans, Eggs) (Onion) 0.8 0.6 0.6 0.75 1.25 0.12 1.6 1.0 3 (Kidney Beans, Onion) (Eggs) 0.6 0.8 0.6 1.00 1.25 0.12 inf 0.5 4 (Eggs) (Kidney Beans, Onion) 0.8 0.6 0.6 0.75 1.25 0.12 1.6 1.0 5 (Onion) (Kidney Beans, Eggs) 0.6 0.8 0.6 1.00 1.25 0.12 inf 0.5 Pandas DataFrames make it easy to filter the results further. Let's say we are ony interested in rules that satisfy the following criteria: at least 2 antecedents a confidence > 0.75 a lift score > 1.2 We could compute the antecedent length as follows: rules[\"antecedent_len\"] = rules[\"antecedents\"].apply(lambda x: len(x)) rules .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } antecedents consequents antecedent support consequent support support confidence lift leverage conviction zhangs_metric antecedent_len 0 (Eggs) (Onion) 0.8 0.6 0.6 0.75 1.25 0.12 1.6 1.0 1 1 (Onion) (Eggs) 0.6 0.8 0.6 1.00 1.25 0.12 inf 0.5 1 2 (Kidney Beans, Eggs) (Onion) 0.8 0.6 0.6 0.75 1.25 0.12 1.6 1.0 2 3 (Kidney Beans, Onion) (Eggs) 0.6 0.8 0.6 1.00 1.25 0.12 inf 0.5 2 4 (Eggs) (Kidney Beans, Onion) 0.8 0.6 0.6 0.75 1.25 0.12 1.6 1.0 1 5 (Onion) (Kidney Beans, Eggs) 0.6 0.8 0.6 1.00 1.25 0.12 inf 0.5 1 Then, we can use pandas' selection syntax as shown below: rules[ (rules['antecedent_len'] >= 2) & (rules['confidence'] > 0.75) & (rules['lift'] > 1.2) ] .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } antecedents consequents antecedent support consequent support support confidence lift leverage conviction zhangs_metric antecedent_len 3 (Kidney Beans, Onion) (Eggs) 0.6 0.8 0.6 1.0 1.25 0.12 inf 0.5 2 Similarly, using the Pandas API, we can select entries based on the \"antecedents\" or \"consequents\" columns: rules[rules['antecedents'] == {'Eggs', 'Kidney Beans'}] .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } antecedents consequents antecedent support consequent support support confidence lift leverage conviction zhangs_metric antecedent_len 2 (Kidney Beans, Eggs) (Onion) 0.8 0.6 0.6 0.75 1.25 0.12 1.6 1.0 2 Frozensets Note that the entries in the \"itemsets\" column are of type frozenset , which is built-in Python type that is similar to a Python set but immutable, which makes it more efficient for certain query or comparison operations (https://docs.python.org/3.6/library/stdtypes.html#frozenset). Since frozenset s are sets, the item order does not matter. I.e., the query rules[rules['antecedents'] == {'Eggs', 'Kidney Beans'}] is equivalent to any of the following three rules[rules['antecedents'] == {'Kidney Beans', 'Eggs'}] rules[rules['antecedents'] == frozenset(('Eggs', 'Kidney Beans'))] rules[rules['antecedents'] == frozenset(('Kidney Beans', 'Eggs'))] Example 3 -- Frequent Itemsets with Incomplete Antecedent and Consequent Information Most metrics computed by association_rules depends on the consequent and antecedent support score of a given rule provided in the frequent itemset input DataFrame. Consider the following example: import pandas as pd dict = {'itemsets': [['177', '176'], ['177', '179'], ['176', '178'], ['176', '179'], ['93', '100'], ['177', '178'], ['177', '176', '178']], 'support':[0.253623, 0.253623, 0.217391, 0.217391, 0.181159, 0.108696, 0.108696]} freq_itemsets = pd.DataFrame(dict) freq_itemsets .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } itemsets support 0 [177, 176] 0.253623 1 [177, 179] 0.253623 2 [176, 178] 0.217391 3 [176, 179] 0.217391 4 [93, 100] 0.181159 5 [177, 178] 0.108696 6 [177, 176, 178] 0.108696 Note that this is a \"cropped\" DataFrame that doesn't contain the support values of the item subsets. This can create problems if we want to compute the association rule metrics for, e.g., 176 => 177 . For example, the confidence is computed as \\text{confidence}(A\\rightarrow C) = \\frac{\\text{support}(A\\rightarrow C)}{\\text{support}(A)}, \\;\\;\\; \\text{range: } [0, 1] But we do not have \\text{support}(A) . All we know about \"A\"'s support is that it is at least 0.253623. In these scenarios, where not all metric's can be computed, due to incomplete input DataFrames, you can use the support_only=True option, which will only compute the support column of a given rule that does not require as much info: \\text{support}(A\\rightarrow C) = \\text{support}(A \\cup C), \\;\\;\\; \\text{range: } [0, 1] \"NaN's\" will be assigned to all other metric columns: from mlxtend.frequent_patterns import association_rules res = association_rules(freq_itemsets, support_only=True, min_threshold=0.1) res .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } antecedents consequents antecedent support consequent support support confidence lift leverage conviction zhangs_metric 0 (176) (177) NaN NaN 0.253623 NaN NaN NaN NaN NaN 1 (177) (176) NaN NaN 0.253623 NaN NaN NaN NaN NaN 2 (179) (177) NaN NaN 0.253623 NaN NaN NaN NaN NaN 3 (177) (179) NaN NaN 0.253623 NaN NaN NaN NaN NaN 4 (178) (176) NaN NaN 0.217391 NaN NaN NaN NaN NaN 5 (176) (178) NaN NaN 0.217391 NaN NaN NaN NaN NaN 6 (179) (176) NaN NaN 0.217391 NaN NaN NaN NaN NaN 7 (176) (179) NaN NaN 0.217391 NaN NaN NaN NaN NaN 8 (100) (93) NaN NaN 0.181159 NaN NaN NaN NaN NaN 9 (93) (100) NaN NaN 0.181159 NaN NaN NaN NaN NaN 10 (178) (177) NaN NaN 0.108696 NaN NaN NaN NaN NaN 11 (177) (178) NaN NaN 0.108696 NaN NaN NaN NaN NaN 12 (178, 176) (177) NaN NaN 0.108696 NaN NaN NaN NaN NaN 13 (178, 177) (176) NaN NaN 0.108696 NaN NaN NaN NaN NaN 14 (177, 176) (178) NaN NaN 0.108696 NaN NaN NaN NaN NaN 15 (178) (177, 176) NaN NaN 0.108696 NaN NaN NaN NaN NaN 16 (176) (178, 177) NaN NaN 0.108696 NaN NaN NaN NaN NaN 17 (177) (178, 176) NaN NaN 0.108696 NaN NaN NaN NaN NaN To clean up the representation, you may want to do the following: res = res[['antecedents', 'consequents', 'support']] res .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } antecedents consequents support 0 (176) (177) 0.253623 1 (177) (176) 0.253623 2 (179) (177) 0.253623 3 (177) (179) 0.253623 4 (178) (176) 0.217391 5 (176) (178) 0.217391 6 (179) (176) 0.217391 7 (176) (179) 0.217391 8 (100) (93) 0.181159 9 (93) (100) 0.181159 10 (178) (177) 0.108696 11 (177) (178) 0.108696 12 (178, 176) (177) 0.108696 13 (178, 177) (176) 0.108696 14 (177, 176) (178) 0.108696 15 (178) (177, 176) 0.108696 16 (176) (178, 177) 0.108696 17 (177) (178, 176) 0.108696 Example 4 -- Pruning Association Rules There is no specific API for pruning. Instead, the pandas API can be used on the resulting data frame to remove individual rows. E.g., suppose we have the following rules: import pandas as pd from mlxtend.preprocessing import TransactionEncoder from mlxtend.frequent_patterns import apriori, fpmax, fpgrowth from mlxtend.frequent_patterns import association_rules dataset = [['Milk', 'Onion', 'Nutmeg', 'Kidney Beans', 'Eggs', 'Yogurt'], ['Dill', 'Onion', 'Nutmeg', 'Kidney Beans', 'Eggs', 'Yogurt'], ['Milk', 'Apple', 'Kidney Beans', 'Eggs'], ['Milk', 'Unicorn', 'Corn', 'Kidney Beans', 'Yogurt'], ['Corn', 'Onion', 'Onion', 'Kidney Beans', 'Ice cream', 'Eggs']] te = TransactionEncoder() te_ary = te.fit(dataset).transform(dataset) df = pd.DataFrame(te_ary, columns=te.columns_) frequent_itemsets = fpgrowth(df, min_support=0.6, use_colnames=True) rules = association_rules(frequent_itemsets, metric=\"lift\", min_threshold=1.2) rules .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } antecedents consequents antecedent support consequent support support confidence lift leverage conviction zhangs_metric 0 (Eggs) (Onion) 0.8 0.6 0.6 0.75 1.25 0.12 1.6 1.0 1 (Onion) (Eggs) 0.6 0.8 0.6 1.00 1.25 0.12 inf 0.5 2 (Kidney Beans, Eggs) (Onion) 0.8 0.6 0.6 0.75 1.25 0.12 1.6 1.0 3 (Kidney Beans, Onion) (Eggs) 0.6 0.8 0.6 1.00 1.25 0.12 inf 0.5 4 (Eggs) (Kidney Beans, Onion) 0.8 0.6 0.6 0.75 1.25 0.12 1.6 1.0 5 (Onion) (Kidney Beans, Eggs) 0.6 0.8 0.6 1.00 1.25 0.12 inf 0.5 and we want to remove the rule \"(Onion, Kidney Beans) -> (Eggs)\". In order to to this, we can define selection masks and remove this row as follows: antecedent_sele = rules['antecedents'] == frozenset({'Onion', 'Kidney Beans'}) # or frozenset({'Kidney Beans', 'Onion'}) consequent_sele = rules['consequents'] == frozenset({'Eggs'}) final_sele = (antecedent_sele & consequent_sele) rules.loc[ ~final_sele ] .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } antecedents consequents antecedent support consequent support support confidence lift leverage conviction zhangs_metric 0 (Eggs) (Onion) 0.8 0.6 0.6 0.75 1.25 0.12 1.6 1.0 1 (Onion) (Eggs) 0.6 0.8 0.6 1.00 1.25 0.12 inf 0.5 2 (Kidney Beans, Eggs) (Onion) 0.8 0.6 0.6 0.75 1.25 0.12 1.6 1.0 4 (Eggs) (Kidney Beans, Onion) 0.8 0.6 0.6 0.75 1.25 0.12 1.6 1.0 5 (Onion) (Kidney Beans, Eggs) 0.6 0.8 0.6 1.00 1.25 0.12 inf 0.5 API","title":"Association rules"},{"location":"user_guide/frequent_patterns/association_rules/#association_rules-association-rules-generation-from-frequent-itemsets","text":"Function to generate association rules from frequent itemsets from mlxtend.frequent_patterns import association_rules","title":"association_rules: Association rules generation from frequent itemsets"},{"location":"user_guide/frequent_patterns/association_rules/#overview","text":"Rule generation is a common task in the mining of frequent patterns. An association rule is an implication expression of the form X \\rightarrow Y , where X and Y are disjoint itemsets [1]. A more concrete example based on consumer behaviour would be \\{Diapers\\} \\rightarrow \\{Beer\\} suggesting that people who buy diapers are also likely to buy beer. To evaluate the \"interest\" of such an association rule, different metrics have been developed. The current implementation make use of the confidence and lift metrics.","title":"Overview"},{"location":"user_guide/frequent_patterns/association_rules/#metrics","text":"The currently supported metrics for evaluating association rules and setting selection thresholds are listed below. Given a rule \"A -> C\", A stands for antecedent and C stands for consequent.","title":"Metrics"},{"location":"user_guide/frequent_patterns/association_rules/#support","text":"\\text{support}(A\\rightarrow C) = \\text{support}(A \\cup C), \\;\\;\\; \\text{range: } [0, 1] introduced in [3] The support metric is defined for itemsets, not assocication rules. The table produced by the association rule mining algorithm contains three different support metrics: 'antecedent support', 'consequent support', and 'support'. Here, 'antecedent support' computes the proportion of transactions that contain the antecedent A, and 'consequent support' computes the support for the itemset of the consequent C. The 'support' metric then computes the support of the combined itemset A \\cup C. Typically, support is used to measure the abundance or frequency (often interpreted as significance or importance) of an itemset in a database. We refer to an itemset as a \"frequent itemset\" if you support is larger than a specified minimum-support threshold. Note that in general, due to the downward closure property, all subsets of a frequent itemset are also frequent.","title":"'support':"},{"location":"user_guide/frequent_patterns/association_rules/#confidence","text":"\\text{confidence}(A\\rightarrow C) = \\frac{\\text{support}(A\\rightarrow C)}{\\text{support}(A)}, \\;\\;\\; \\text{range: } [0, 1] introduced in [3] The confidence of a rule A->C is the probability of seeing the consequent in a transaction given that it also contains the antecedent. Note that the metric is not symmetric or directed; for instance, the confidence for A->C is different than the confidence for C->A. The confidence is 1 (maximal) for a rule A->C if the consequent and antecedent always occur together.","title":"'confidence':"},{"location":"user_guide/frequent_patterns/association_rules/#lift","text":"\\text{lift}(A\\rightarrow C) = \\frac{\\text{confidence}(A\\rightarrow C)}{\\text{support}(C)}, \\;\\;\\; \\text{range: } [0, \\infty] introduced in [4] The lift metric is commonly used to measure how much more often the antecedent and consequent of a rule A->C occur together than we would expect if they were statistically independent. If A and C are independent, the Lift score will be exactly 1.","title":"'lift':"},{"location":"user_guide/frequent_patterns/association_rules/#leverage","text":"\\text{levarage}(A\\rightarrow C) = \\text{support}(A\\rightarrow C) - \\text{support}(A) \\times \\text{support}(C), \\;\\;\\; \\text{range: } [-1, 1] introduced in [5] Leverage computes the difference between the observed frequency of A and C appearing together and the frequency that would be expected if A and C were independent. A leverage value of 0 indicates independence.","title":"'leverage':"},{"location":"user_guide/frequent_patterns/association_rules/#conviction","text":"\\text{conviction}(A\\rightarrow C) = \\frac{1 - \\text{support}(C)}{1 - \\text{confidence}(A\\rightarrow C)}, \\;\\;\\; \\text{range: } [0, \\infty] introduced in [6] A high conviction value means that the consequent is highly depending on the antecedent. For instance, in the case of a perfect confidence score, the denominator becomes 0 (due to 1 - 1) for which the conviction score is defined as 'inf'. Similar to lift, if items are independent, the conviction is 1.","title":"'conviction':"},{"location":"user_guide/frequent_patterns/association_rules/#zhangs_metric","text":"\\text{zhangs metric}(A\\rightarrow C) = \\frac{\\text{confidence}(A\\rightarrow C) - \\text{confidence}(A'\\rightarrow C)}{Max[ \\text{confidence}(A\\rightarrow C) , \\text{confidence}(A'\\rightarrow C)]}, \\;\\;\\; \\text{range: } [-1, 1] introduced in [7] Measures both association and dissociation. Value ranges between -1 and 1. A positive value (>0) indicates Association and negative value indicated dissociation.","title":"'zhangs_metric':"},{"location":"user_guide/frequent_patterns/association_rules/#references","text":"[1] Tan, Steinbach, Kumar. Introduction to Data Mining. Pearson New International Edition. Harlow: Pearson Education Ltd., 2014. (pp. 327-414). [2] Michael Hahsler, https://michael.hahsler.net/research/association_rules/measures.html [3] R. Agrawal, T. Imielinski, and A. Swami. Mining associations between sets of items in large databases. In Proc. of the ACM SIGMOD Int'l Conference on Management of Data, pages 207-216, Washington D.C., May 1993 [4] S. Brin, R. Motwani, J. D. Ullman, and S. Tsur. Dynamic itemset counting and implication rules for market basket data [5] Piatetsky-Shapiro, G., Discovery, analysis, and presentation of strong rules. Knowledge Discovery in Databases, 1991: p. 229-248. [6] Sergey Brin, Rajeev Motwani, Jeffrey D. Ullman, and Shalom Turk. Dynamic itemset counting and implication rules for market basket data. In SIGMOD 1997, Proceedings ACM SIGMOD International Conference on Management of Data, pages 255-264, Tucson, Arizona, USA, May 1997 [7] Xiaowei Yan , Chengqi Zhang & Shichao Zhang (2009) CONFIDENCE METRICS FOR ASSOCIATION RULE MINING, Applied Artificial Intelligence, 23:8, 713-737 https://www.tandfonline.com/doi/pdf/10.1080/08839510903208062.","title":"References"},{"location":"user_guide/frequent_patterns/association_rules/#example-1-generating-association-rules-from-frequent-itemsets","text":"The generate_rules takes dataframes of frequent itemsets as produced by the apriori , fpgrowth , or fpmax functions in mlxtend.association . To demonstrate the usage of the generate_rules method, we first create a pandas DataFrame of frequent itemsets as generated by the fpgrowth function: import pandas as pd from mlxtend.preprocessing import TransactionEncoder from mlxtend.frequent_patterns import apriori, fpmax, fpgrowth dataset = [['Milk', 'Onion', 'Nutmeg', 'Kidney Beans', 'Eggs', 'Yogurt'], ['Dill', 'Onion', 'Nutmeg', 'Kidney Beans', 'Eggs', 'Yogurt'], ['Milk', 'Apple', 'Kidney Beans', 'Eggs'], ['Milk', 'Unicorn', 'Corn', 'Kidney Beans', 'Yogurt'], ['Corn', 'Onion', 'Onion', 'Kidney Beans', 'Ice cream', 'Eggs']] te = TransactionEncoder() te_ary = te.fit(dataset).transform(dataset) df = pd.DataFrame(te_ary, columns=te.columns_) frequent_itemsets = fpgrowth(df, min_support=0.6, use_colnames=True) ### alternatively: #frequent_itemsets = apriori(df, min_support=0.6, use_colnames=True) #frequent_itemsets = fpmax(df, min_support=0.6, use_colnames=True) frequent_itemsets .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } support itemsets 0 1.0 (Kidney Beans) 1 0.8 (Eggs) 2 0.6 (Yogurt) 3 0.6 (Onion) 4 0.6 (Milk) 5 0.8 (Kidney Beans, Eggs) 6 0.6 (Kidney Beans, Yogurt) 7 0.6 (Eggs, Onion) 8 0.6 (Kidney Beans, Onion) 9 0.6 (Eggs, Kidney Beans, Onion) 10 0.6 (Kidney Beans, Milk) The generate_rules() function allows you to (1) specify your metric of interest and (2) the according threshold. Currently implemented measures are confidence and lift . Let's say you are interested in rules derived from the frequent itemsets only if the level of confidence is above the 70 percent threshold ( min_threshold=0.7 ): from mlxtend.frequent_patterns import association_rules association_rules(frequent_itemsets, metric=\"confidence\", min_threshold=0.7) .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } antecedents consequents antecedent support consequent support support confidence lift leverage conviction zhangs_metric 0 (Kidney Beans) (Eggs) 1.0 0.8 0.8 0.80 1.00 0.00 1.0 0.0 1 (Eggs) (Kidney Beans) 0.8 1.0 0.8 1.00 1.00 0.00 inf 0.0 2 (Yogurt) (Kidney Beans) 0.6 1.0 0.6 1.00 1.00 0.00 inf 0.0 3 (Eggs) (Onion) 0.8 0.6 0.6 0.75 1.25 0.12 1.6 1.0 4 (Onion) (Eggs) 0.6 0.8 0.6 1.00 1.25 0.12 inf 0.5 5 (Onion) (Kidney Beans) 0.6 1.0 0.6 1.00 1.00 0.00 inf 0.0 6 (Kidney Beans, Eggs) (Onion) 0.8 0.6 0.6 0.75 1.25 0.12 1.6 1.0 7 (Onion, Eggs) (Kidney Beans) 0.6 1.0 0.6 1.00 1.00 0.00 inf 0.0 8 (Kidney Beans, Onion) (Eggs) 0.6 0.8 0.6 1.00 1.25 0.12 inf 0.5 9 (Eggs) (Kidney Beans, Onion) 0.8 0.6 0.6 0.75 1.25 0.12 1.6 1.0 10 (Onion) (Kidney Beans, Eggs) 0.6 0.8 0.6 1.00 1.25 0.12 inf 0.5 11 (Milk) (Kidney Beans) 0.6 1.0 0.6 1.00 1.00 0.00 inf 0.0","title":"Example 1 -- Generating Association Rules from Frequent Itemsets"},{"location":"user_guide/frequent_patterns/association_rules/#example-2-rule-generation-and-selection-criteria","text":"If you are interested in rules according to a different metric of interest, you can simply adjust the metric and min_threshold arguments . E.g. if you are only interested in rules that have a lift score of >= 1.2, you would do the following: rules = association_rules(frequent_itemsets, metric=\"lift\", min_threshold=1.2) rules .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } antecedents consequents antecedent support consequent support support confidence lift leverage conviction zhangs_metric 0 (Eggs) (Onion) 0.8 0.6 0.6 0.75 1.25 0.12 1.6 1.0 1 (Onion) (Eggs) 0.6 0.8 0.6 1.00 1.25 0.12 inf 0.5 2 (Kidney Beans, Eggs) (Onion) 0.8 0.6 0.6 0.75 1.25 0.12 1.6 1.0 3 (Kidney Beans, Onion) (Eggs) 0.6 0.8 0.6 1.00 1.25 0.12 inf 0.5 4 (Eggs) (Kidney Beans, Onion) 0.8 0.6 0.6 0.75 1.25 0.12 1.6 1.0 5 (Onion) (Kidney Beans, Eggs) 0.6 0.8 0.6 1.00 1.25 0.12 inf 0.5 Pandas DataFrames make it easy to filter the results further. Let's say we are ony interested in rules that satisfy the following criteria: at least 2 antecedents a confidence > 0.75 a lift score > 1.2 We could compute the antecedent length as follows: rules[\"antecedent_len\"] = rules[\"antecedents\"].apply(lambda x: len(x)) rules .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } antecedents consequents antecedent support consequent support support confidence lift leverage conviction zhangs_metric antecedent_len 0 (Eggs) (Onion) 0.8 0.6 0.6 0.75 1.25 0.12 1.6 1.0 1 1 (Onion) (Eggs) 0.6 0.8 0.6 1.00 1.25 0.12 inf 0.5 1 2 (Kidney Beans, Eggs) (Onion) 0.8 0.6 0.6 0.75 1.25 0.12 1.6 1.0 2 3 (Kidney Beans, Onion) (Eggs) 0.6 0.8 0.6 1.00 1.25 0.12 inf 0.5 2 4 (Eggs) (Kidney Beans, Onion) 0.8 0.6 0.6 0.75 1.25 0.12 1.6 1.0 1 5 (Onion) (Kidney Beans, Eggs) 0.6 0.8 0.6 1.00 1.25 0.12 inf 0.5 1 Then, we can use pandas' selection syntax as shown below: rules[ (rules['antecedent_len'] >= 2) & (rules['confidence'] > 0.75) & (rules['lift'] > 1.2) ] .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } antecedents consequents antecedent support consequent support support confidence lift leverage conviction zhangs_metric antecedent_len 3 (Kidney Beans, Onion) (Eggs) 0.6 0.8 0.6 1.0 1.25 0.12 inf 0.5 2 Similarly, using the Pandas API, we can select entries based on the \"antecedents\" or \"consequents\" columns: rules[rules['antecedents'] == {'Eggs', 'Kidney Beans'}] .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } antecedents consequents antecedent support consequent support support confidence lift leverage conviction zhangs_metric antecedent_len 2 (Kidney Beans, Eggs) (Onion) 0.8 0.6 0.6 0.75 1.25 0.12 1.6 1.0 2 Frozensets Note that the entries in the \"itemsets\" column are of type frozenset , which is built-in Python type that is similar to a Python set but immutable, which makes it more efficient for certain query or comparison operations (https://docs.python.org/3.6/library/stdtypes.html#frozenset). Since frozenset s are sets, the item order does not matter. I.e., the query rules[rules['antecedents'] == {'Eggs', 'Kidney Beans'}] is equivalent to any of the following three rules[rules['antecedents'] == {'Kidney Beans', 'Eggs'}] rules[rules['antecedents'] == frozenset(('Eggs', 'Kidney Beans'))] rules[rules['antecedents'] == frozenset(('Kidney Beans', 'Eggs'))]","title":"Example 2 -- Rule Generation and Selection Criteria"},{"location":"user_guide/frequent_patterns/association_rules/#example-3-frequent-itemsets-with-incomplete-antecedent-and-consequent-information","text":"Most metrics computed by association_rules depends on the consequent and antecedent support score of a given rule provided in the frequent itemset input DataFrame. Consider the following example: import pandas as pd dict = {'itemsets': [['177', '176'], ['177', '179'], ['176', '178'], ['176', '179'], ['93', '100'], ['177', '178'], ['177', '176', '178']], 'support':[0.253623, 0.253623, 0.217391, 0.217391, 0.181159, 0.108696, 0.108696]} freq_itemsets = pd.DataFrame(dict) freq_itemsets .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } itemsets support 0 [177, 176] 0.253623 1 [177, 179] 0.253623 2 [176, 178] 0.217391 3 [176, 179] 0.217391 4 [93, 100] 0.181159 5 [177, 178] 0.108696 6 [177, 176, 178] 0.108696 Note that this is a \"cropped\" DataFrame that doesn't contain the support values of the item subsets. This can create problems if we want to compute the association rule metrics for, e.g., 176 => 177 . For example, the confidence is computed as \\text{confidence}(A\\rightarrow C) = \\frac{\\text{support}(A\\rightarrow C)}{\\text{support}(A)}, \\;\\;\\; \\text{range: } [0, 1] But we do not have \\text{support}(A) . All we know about \"A\"'s support is that it is at least 0.253623. In these scenarios, where not all metric's can be computed, due to incomplete input DataFrames, you can use the support_only=True option, which will only compute the support column of a given rule that does not require as much info: \\text{support}(A\\rightarrow C) = \\text{support}(A \\cup C), \\;\\;\\; \\text{range: } [0, 1] \"NaN's\" will be assigned to all other metric columns: from mlxtend.frequent_patterns import association_rules res = association_rules(freq_itemsets, support_only=True, min_threshold=0.1) res .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } antecedents consequents antecedent support consequent support support confidence lift leverage conviction zhangs_metric 0 (176) (177) NaN NaN 0.253623 NaN NaN NaN NaN NaN 1 (177) (176) NaN NaN 0.253623 NaN NaN NaN NaN NaN 2 (179) (177) NaN NaN 0.253623 NaN NaN NaN NaN NaN 3 (177) (179) NaN NaN 0.253623 NaN NaN NaN NaN NaN 4 (178) (176) NaN NaN 0.217391 NaN NaN NaN NaN NaN 5 (176) (178) NaN NaN 0.217391 NaN NaN NaN NaN NaN 6 (179) (176) NaN NaN 0.217391 NaN NaN NaN NaN NaN 7 (176) (179) NaN NaN 0.217391 NaN NaN NaN NaN NaN 8 (100) (93) NaN NaN 0.181159 NaN NaN NaN NaN NaN 9 (93) (100) NaN NaN 0.181159 NaN NaN NaN NaN NaN 10 (178) (177) NaN NaN 0.108696 NaN NaN NaN NaN NaN 11 (177) (178) NaN NaN 0.108696 NaN NaN NaN NaN NaN 12 (178, 176) (177) NaN NaN 0.108696 NaN NaN NaN NaN NaN 13 (178, 177) (176) NaN NaN 0.108696 NaN NaN NaN NaN NaN 14 (177, 176) (178) NaN NaN 0.108696 NaN NaN NaN NaN NaN 15 (178) (177, 176) NaN NaN 0.108696 NaN NaN NaN NaN NaN 16 (176) (178, 177) NaN NaN 0.108696 NaN NaN NaN NaN NaN 17 (177) (178, 176) NaN NaN 0.108696 NaN NaN NaN NaN NaN To clean up the representation, you may want to do the following: res = res[['antecedents', 'consequents', 'support']] res .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } antecedents consequents support 0 (176) (177) 0.253623 1 (177) (176) 0.253623 2 (179) (177) 0.253623 3 (177) (179) 0.253623 4 (178) (176) 0.217391 5 (176) (178) 0.217391 6 (179) (176) 0.217391 7 (176) (179) 0.217391 8 (100) (93) 0.181159 9 (93) (100) 0.181159 10 (178) (177) 0.108696 11 (177) (178) 0.108696 12 (178, 176) (177) 0.108696 13 (178, 177) (176) 0.108696 14 (177, 176) (178) 0.108696 15 (178) (177, 176) 0.108696 16 (176) (178, 177) 0.108696 17 (177) (178, 176) 0.108696","title":"Example 3 -- Frequent Itemsets with Incomplete Antecedent and Consequent Information"},{"location":"user_guide/frequent_patterns/association_rules/#example-4-pruning-association-rules","text":"There is no specific API for pruning. Instead, the pandas API can be used on the resulting data frame to remove individual rows. E.g., suppose we have the following rules: import pandas as pd from mlxtend.preprocessing import TransactionEncoder from mlxtend.frequent_patterns import apriori, fpmax, fpgrowth from mlxtend.frequent_patterns import association_rules dataset = [['Milk', 'Onion', 'Nutmeg', 'Kidney Beans', 'Eggs', 'Yogurt'], ['Dill', 'Onion', 'Nutmeg', 'Kidney Beans', 'Eggs', 'Yogurt'], ['Milk', 'Apple', 'Kidney Beans', 'Eggs'], ['Milk', 'Unicorn', 'Corn', 'Kidney Beans', 'Yogurt'], ['Corn', 'Onion', 'Onion', 'Kidney Beans', 'Ice cream', 'Eggs']] te = TransactionEncoder() te_ary = te.fit(dataset).transform(dataset) df = pd.DataFrame(te_ary, columns=te.columns_) frequent_itemsets = fpgrowth(df, min_support=0.6, use_colnames=True) rules = association_rules(frequent_itemsets, metric=\"lift\", min_threshold=1.2) rules .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } antecedents consequents antecedent support consequent support support confidence lift leverage conviction zhangs_metric 0 (Eggs) (Onion) 0.8 0.6 0.6 0.75 1.25 0.12 1.6 1.0 1 (Onion) (Eggs) 0.6 0.8 0.6 1.00 1.25 0.12 inf 0.5 2 (Kidney Beans, Eggs) (Onion) 0.8 0.6 0.6 0.75 1.25 0.12 1.6 1.0 3 (Kidney Beans, Onion) (Eggs) 0.6 0.8 0.6 1.00 1.25 0.12 inf 0.5 4 (Eggs) (Kidney Beans, Onion) 0.8 0.6 0.6 0.75 1.25 0.12 1.6 1.0 5 (Onion) (Kidney Beans, Eggs) 0.6 0.8 0.6 1.00 1.25 0.12 inf 0.5 and we want to remove the rule \"(Onion, Kidney Beans) -> (Eggs)\". In order to to this, we can define selection masks and remove this row as follows: antecedent_sele = rules['antecedents'] == frozenset({'Onion', 'Kidney Beans'}) # or frozenset({'Kidney Beans', 'Onion'}) consequent_sele = rules['consequents'] == frozenset({'Eggs'}) final_sele = (antecedent_sele & consequent_sele) rules.loc[ ~final_sele ] .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } antecedents consequents antecedent support consequent support support confidence lift leverage conviction zhangs_metric 0 (Eggs) (Onion) 0.8 0.6 0.6 0.75 1.25 0.12 1.6 1.0 1 (Onion) (Eggs) 0.6 0.8 0.6 1.00 1.25 0.12 inf 0.5 2 (Kidney Beans, Eggs) (Onion) 0.8 0.6 0.6 0.75 1.25 0.12 1.6 1.0 4 (Eggs) (Kidney Beans, Onion) 0.8 0.6 0.6 0.75 1.25 0.12 1.6 1.0 5 (Onion) (Kidney Beans, Eggs) 0.6 0.8 0.6 1.00 1.25 0.12 inf 0.5","title":"Example 4 -- Pruning Association Rules"},{"location":"user_guide/frequent_patterns/association_rules/#api","text":"","title":"API"},{"location":"user_guide/frequent_patterns/fpgrowth/","text":"fpgrowth: Frequent itemsets via the FP-growth algorithm Function implementing FP-Growth to extract frequent itemsets for association rule mining from mlxtend.frequent_patterns import fpgrowth Overview FP-Growth [1] is an algorithm for extracting frequent itemsets with applications in association rule learning that emerged as a popular alternative to the established Apriori algorithm [2]. In general, the algorithm has been designed to operate on databases containing transactions, such as purchases by customers of a store. An itemset is considered as \"frequent\" if it meets a user-specified support threshold. For instance, if the support threshold is set to 0.5 (50%), a frequent itemset is defined as a set of items that occur together in at least 50% of all transactions in the database. In particular, and what makes it different from the Apriori frequent pattern mining algorithm, FP-Growth is an frequent pattern mining algorithm that does not require candidate generation. Internally, it uses a so-called FP-tree (frequent pattern tree) datastrucure without generating the candidate sets explicitly, which makes it particularly attractive for large datasets. References [1] Han, Jiawei, Jian Pei, Yiwen Yin, and Runying Mao. \"Mining frequent patterns without candidate generation. \" A frequent-pattern tree approach. \" Data mining and knowledge discovery 8, no. 1 (2004): 53-87. [2] Agrawal, Rakesh, and Ramakrishnan Srikant. \" Fast algorithms for mining association rules .\" Proc. 20th int. conf. very large data bases, VLDB. Vol. 1215. 1994. Related FP-Max Apriori Example 1 -- Generating Frequent Itemsets The fpgrowth function expects data in a one-hot encoded pandas DataFrame. Suppose we have the following transaction data: dataset = [['Milk', 'Onion', 'Nutmeg', 'Kidney Beans', 'Eggs', 'Yogurt'], ['Dill', 'Onion', 'Nutmeg', 'Kidney Beans', 'Eggs', 'Yogurt'], ['Milk', 'Apple', 'Kidney Beans', 'Eggs'], ['Milk', 'Unicorn', 'Corn', 'Kidney Beans', 'Yogurt'], ['Corn', 'Onion', 'Onion', 'Kidney Beans', 'Ice cream', 'Eggs']] We can transform it into the right format via the TransactionEncoder as follows: import pandas as pd from mlxtend.preprocessing import TransactionEncoder te = TransactionEncoder() te_ary = te.fit(dataset).transform(dataset) df = pd.DataFrame(te_ary, columns=te.columns_) df .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } Apple Corn Dill Eggs Ice cream Kidney Beans Milk Nutmeg Onion Unicorn Yogurt 0 False False False True False True True True True False True 1 False False True True False True False True True False True 2 True False False True False True True False False False False 3 False True False False False True True False False True True 4 False True False True True True False False True False False Now, let us return the items and itemsets with at least 60% support: from mlxtend.frequent_patterns import fpgrowth fpgrowth(df, min_support=0.6) .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } support itemsets 0 1.0 (5) 1 0.8 (3) 2 0.6 (10) 3 0.6 (8) 4 0.6 (6) 5 0.8 (3, 5) 6 0.6 (10, 5) 7 0.6 (8, 3) 8 0.6 (8, 5) 9 0.6 (8, 3, 5) 10 0.6 (5, 6) By default, fpgrowth returns the column indices of the items, which may be useful in downstream operations such as association rule mining. For better readability, we can set use_colnames=True to convert these integer values into the respective item names: fpgrowth(df, min_support=0.6, use_colnames=True) .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } support itemsets 0 1.0 (Kidney Beans) 1 0.8 (Eggs) 2 0.6 (Yogurt) 3 0.6 (Onion) 4 0.6 (Milk) 5 0.8 (Eggs, Kidney Beans) 6 0.6 (Yogurt, Kidney Beans) 7 0.6 (Eggs, Onion) 8 0.6 (Onion, Kidney Beans) 9 0.6 (Eggs, Onion, Kidney Beans) 10 0.6 (Milk, Kidney Beans) Example 2 -- Apriori versus FPGrowth Since FP-Growth doesn't require creating candidate sets explicitly, it can be magnitudes faster than the alternative Apriori algorithm. For instance, the following cells compare the performance of the Apriori algorithm to the performance of FP-Growth -- even in this very simple toy dataset scenario, FP-Growth is about 5 times faster. import pandas as pd from mlxtend.preprocessing import TransactionEncoder te = TransactionEncoder() te_ary = te.fit(dataset).transform(dataset) df = pd.DataFrame(te_ary, columns=te.columns_) from mlxtend.frequent_patterns import apriori %timeit -n 100 -r 10 apriori(df, min_support=0.6) 850 \u00b5s \u00b1 39.3 \u00b5s per loop (mean \u00b1 std. dev. of 10 runs, 100 loops each) %timeit -n 100 -r 10 apriori(df, min_support=0.6, low_memory=True) 941 \u00b5s \u00b1 30.6 \u00b5s per loop (mean \u00b1 std. dev. of 10 runs, 100 loops each) from mlxtend.frequent_patterns import fpgrowth %timeit -n 100 -r 10 fpgrowth(df, min_support=0.6) 320 \u00b5s \u00b1 9.21 \u00b5s per loop (mean \u00b1 std. dev. of 10 runs, 100 loops each) More Examples Please note that since the fpgrowth function is a drop-in replacement for apriori , it comes with the same set of function arguments and return arguments. Thus, for more examples, please see the apriori documentation. API fpgrowth(df, min_support=0.5, use_colnames=False, max_len=None, verbose=0) Get frequent itemsets from a one-hot DataFrame Parameters df : pandas DataFrame pandas DataFrame the encoded format. Also supports DataFrames with sparse data; for more info, please see https://pandas.pydata.org/pandas-docs/stable/user_guide/sparse.html#sparse-data-structures. Please note that the old pandas SparseDataFrame format is no longer supported in mlxtend >= 0.17.2. The allowed values are either 0/1 or True/False. For example, Apple Bananas Beer Chicken Milk Rice 0 True False True True False True 1 True False True False False True 2 True False True False False False 3 True True False False False False 4 False False True True True True 5 False False True False True True 6 False False True False True False 7 True True False False False False min_support : float (default: 0.5) A float between 0 and 1 for minimum support of the itemsets returned. The support is computed as the fraction transactions_where_item(s)_occur / total_transactions. use_colnames : bool (default: False) If true, uses the DataFrames' column names in the returned DataFrame instead of column indices. max_len : int (default: None) Maximum length of the itemsets generated. If None (default) all possible itemsets lengths are evaluated. verbose : int (default: 0) Shows the stages of conditional tree generation. Returns pandas DataFrame with columns ['support', 'itemsets'] of all itemsets that are >= min_support and < than max_len (if max_len is not None). Each itemset in the 'itemsets' column is of type frozenset , which is a Python built-in type that behaves similarly to sets except that it is immutable (For more info, see https://docs.python.org/3.6/library/stdtypes.html#frozenset). Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/frequent_patterns/fpgrowth/","title":"Fpgrowth"},{"location":"user_guide/frequent_patterns/fpgrowth/#fpgrowth-frequent-itemsets-via-the-fp-growth-algorithm","text":"Function implementing FP-Growth to extract frequent itemsets for association rule mining from mlxtend.frequent_patterns import fpgrowth","title":"fpgrowth: Frequent itemsets via the FP-growth algorithm"},{"location":"user_guide/frequent_patterns/fpgrowth/#overview","text":"FP-Growth [1] is an algorithm for extracting frequent itemsets with applications in association rule learning that emerged as a popular alternative to the established Apriori algorithm [2]. In general, the algorithm has been designed to operate on databases containing transactions, such as purchases by customers of a store. An itemset is considered as \"frequent\" if it meets a user-specified support threshold. For instance, if the support threshold is set to 0.5 (50%), a frequent itemset is defined as a set of items that occur together in at least 50% of all transactions in the database. In particular, and what makes it different from the Apriori frequent pattern mining algorithm, FP-Growth is an frequent pattern mining algorithm that does not require candidate generation. Internally, it uses a so-called FP-tree (frequent pattern tree) datastrucure without generating the candidate sets explicitly, which makes it particularly attractive for large datasets.","title":"Overview"},{"location":"user_guide/frequent_patterns/fpgrowth/#references","text":"[1] Han, Jiawei, Jian Pei, Yiwen Yin, and Runying Mao. \"Mining frequent patterns without candidate generation. \" A frequent-pattern tree approach. \" Data mining and knowledge discovery 8, no. 1 (2004): 53-87. [2] Agrawal, Rakesh, and Ramakrishnan Srikant. \" Fast algorithms for mining association rules .\" Proc. 20th int. conf. very large data bases, VLDB. Vol. 1215. 1994.","title":"References"},{"location":"user_guide/frequent_patterns/fpgrowth/#related","text":"FP-Max Apriori","title":"Related"},{"location":"user_guide/frequent_patterns/fpgrowth/#example-1-generating-frequent-itemsets","text":"The fpgrowth function expects data in a one-hot encoded pandas DataFrame. Suppose we have the following transaction data: dataset = [['Milk', 'Onion', 'Nutmeg', 'Kidney Beans', 'Eggs', 'Yogurt'], ['Dill', 'Onion', 'Nutmeg', 'Kidney Beans', 'Eggs', 'Yogurt'], ['Milk', 'Apple', 'Kidney Beans', 'Eggs'], ['Milk', 'Unicorn', 'Corn', 'Kidney Beans', 'Yogurt'], ['Corn', 'Onion', 'Onion', 'Kidney Beans', 'Ice cream', 'Eggs']] We can transform it into the right format via the TransactionEncoder as follows: import pandas as pd from mlxtend.preprocessing import TransactionEncoder te = TransactionEncoder() te_ary = te.fit(dataset).transform(dataset) df = pd.DataFrame(te_ary, columns=te.columns_) df .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } Apple Corn Dill Eggs Ice cream Kidney Beans Milk Nutmeg Onion Unicorn Yogurt 0 False False False True False True True True True False True 1 False False True True False True False True True False True 2 True False False True False True True False False False False 3 False True False False False True True False False True True 4 False True False True True True False False True False False Now, let us return the items and itemsets with at least 60% support: from mlxtend.frequent_patterns import fpgrowth fpgrowth(df, min_support=0.6) .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } support itemsets 0 1.0 (5) 1 0.8 (3) 2 0.6 (10) 3 0.6 (8) 4 0.6 (6) 5 0.8 (3, 5) 6 0.6 (10, 5) 7 0.6 (8, 3) 8 0.6 (8, 5) 9 0.6 (8, 3, 5) 10 0.6 (5, 6) By default, fpgrowth returns the column indices of the items, which may be useful in downstream operations such as association rule mining. For better readability, we can set use_colnames=True to convert these integer values into the respective item names: fpgrowth(df, min_support=0.6, use_colnames=True) .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } support itemsets 0 1.0 (Kidney Beans) 1 0.8 (Eggs) 2 0.6 (Yogurt) 3 0.6 (Onion) 4 0.6 (Milk) 5 0.8 (Eggs, Kidney Beans) 6 0.6 (Yogurt, Kidney Beans) 7 0.6 (Eggs, Onion) 8 0.6 (Onion, Kidney Beans) 9 0.6 (Eggs, Onion, Kidney Beans) 10 0.6 (Milk, Kidney Beans)","title":"Example 1 -- Generating Frequent Itemsets"},{"location":"user_guide/frequent_patterns/fpgrowth/#example-2-apriori-versus-fpgrowth","text":"Since FP-Growth doesn't require creating candidate sets explicitly, it can be magnitudes faster than the alternative Apriori algorithm. For instance, the following cells compare the performance of the Apriori algorithm to the performance of FP-Growth -- even in this very simple toy dataset scenario, FP-Growth is about 5 times faster. import pandas as pd from mlxtend.preprocessing import TransactionEncoder te = TransactionEncoder() te_ary = te.fit(dataset).transform(dataset) df = pd.DataFrame(te_ary, columns=te.columns_) from mlxtend.frequent_patterns import apriori %timeit -n 100 -r 10 apriori(df, min_support=0.6) 850 \u00b5s \u00b1 39.3 \u00b5s per loop (mean \u00b1 std. dev. of 10 runs, 100 loops each) %timeit -n 100 -r 10 apriori(df, min_support=0.6, low_memory=True) 941 \u00b5s \u00b1 30.6 \u00b5s per loop (mean \u00b1 std. dev. of 10 runs, 100 loops each) from mlxtend.frequent_patterns import fpgrowth %timeit -n 100 -r 10 fpgrowth(df, min_support=0.6) 320 \u00b5s \u00b1 9.21 \u00b5s per loop (mean \u00b1 std. dev. of 10 runs, 100 loops each)","title":"Example 2 -- Apriori versus FPGrowth"},{"location":"user_guide/frequent_patterns/fpgrowth/#more-examples","text":"Please note that since the fpgrowth function is a drop-in replacement for apriori , it comes with the same set of function arguments and return arguments. Thus, for more examples, please see the apriori documentation.","title":"More Examples"},{"location":"user_guide/frequent_patterns/fpgrowth/#api","text":"fpgrowth(df, min_support=0.5, use_colnames=False, max_len=None, verbose=0) Get frequent itemsets from a one-hot DataFrame Parameters df : pandas DataFrame pandas DataFrame the encoded format. Also supports DataFrames with sparse data; for more info, please see https://pandas.pydata.org/pandas-docs/stable/user_guide/sparse.html#sparse-data-structures. Please note that the old pandas SparseDataFrame format is no longer supported in mlxtend >= 0.17.2. The allowed values are either 0/1 or True/False. For example, Apple Bananas Beer Chicken Milk Rice 0 True False True True False True 1 True False True False False True 2 True False True False False False 3 True True False False False False 4 False False True True True True 5 False False True False True True 6 False False True False True False 7 True True False False False False min_support : float (default: 0.5) A float between 0 and 1 for minimum support of the itemsets returned. The support is computed as the fraction transactions_where_item(s)_occur / total_transactions. use_colnames : bool (default: False) If true, uses the DataFrames' column names in the returned DataFrame instead of column indices. max_len : int (default: None) Maximum length of the itemsets generated. If None (default) all possible itemsets lengths are evaluated. verbose : int (default: 0) Shows the stages of conditional tree generation. Returns pandas DataFrame with columns ['support', 'itemsets'] of all itemsets that are >= min_support and < than max_len (if max_len is not None). Each itemset in the 'itemsets' column is of type frozenset , which is a Python built-in type that behaves similarly to sets except that it is immutable (For more info, see https://docs.python.org/3.6/library/stdtypes.html#frozenset). Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/frequent_patterns/fpgrowth/","title":"API"},{"location":"user_guide/frequent_patterns/fpmax/","text":"fpmax: Maximal itemsets via the FP-Max algorithm Function implementing FP-Max to extract maximal itemsets for association rule mining from mlxtend.frequent_patterns import fpmax Overview The Apriori algorithm is among the first and most popular algorithms for frequent itemset generation (frequent itemsets are then used for association rule mining). However, the runtime of Apriori can be quite large, especially for datasets with a large number of unique items, as the runtime grows exponentially depending on the number of unique items. In contrast to Apriori, FP-Growth is a frequent pattern generation algorithm that inserts items into a pattern search tree, which allows it to have a linear increase in runtime with respect to the number of unique items or entries. FP-Max is a variant of FP-Growth, which focuses on obtaining maximal itemsets. An itemset X is said to maximal if X is frequent and there exists no frequent super-pattern containing X. In other words, a frequent pattern X cannot be sub-pattern of larger frequent pattern to qualify for the definition maximal itemset . References [1] Grahne, G., & Zhu, J. (2003, November). Efficiently using prefix-trees in mining frequent itemsets. In FIMI (Vol. 90). Related FP-Growth Apriori Example 1 -- Maximal Itemsets The fpmax function expects data in a one-hot encoded pandas DataFrame. Suppose we have the following transaction data: dataset = [['Milk', 'Onion', 'Nutmeg', 'Kidney Beans', 'Eggs', 'Yogurt'], ['Dill', 'Onion', 'Nutmeg', 'Kidney Beans', 'Eggs', 'Yogurt'], ['Milk', 'Apple', 'Kidney Beans', 'Eggs'], ['Milk', 'Unicorn', 'Corn', 'Kidney Beans', 'Yogurt'], ['Corn', 'Onion', 'Onion', 'Kidney Beans', 'Ice cream', 'Eggs']] We can transform it into the right format via the TransactionEncoder as follows: import pandas as pd from mlxtend.preprocessing import TransactionEncoder te = TransactionEncoder() te_ary = te.fit(dataset).transform(dataset) df = pd.DataFrame(te_ary, columns=te.columns_) df .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } Apple Corn Dill Eggs Ice cream Kidney Beans Milk Nutmeg Onion Unicorn Yogurt 0 False False False True False True True True True False True 1 False False True True False True False True True False True 2 True False False True False True True False False False False 3 False True False False False True True False False True True 4 False True False True True True False False True False False Now, let us return the items and itemsets with at least 60% support: from mlxtend.frequent_patterns import fpmax fpmax(df, min_support=0.6) .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } support itemsets 0 0.6 (5, 6) 1 0.6 (8, 3, 5) 2 0.6 (10, 5) By default, fpmax returns the column indices of the items, which may be useful in downstream operations such as association rule mining. For better readability, we can set use_colnames=True to convert these integer values into the respective item names: fpmax(df, min_support=0.6, use_colnames=True) .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } support itemsets 0 0.6 (Kidney Beans, Milk) 1 0.6 (Onion, Eggs, Kidney Beans) 2 0.6 (Kidney Beans, Yogurt) More Examples Please note that since the fpmax function is a drop-in replacement for fpgrowth and apriori , it comes with the same set of function arguments and return arguments. Thus, for more examples, please see the apriori documentation. API fpmax(df, min_support=0.5, use_colnames=False, max_len=None, verbose=0) Get maximal frequent itemsets from a one-hot DataFrame Parameters df : pandas DataFrame pandas DataFrame the encoded format. Also supports DataFrames with sparse data; for more info, please see (https://pandas.pydata.org/pandas-docs/stable/ user_guide/sparse.html#sparse-data-structures) Please note that the old pandas SparseDataFrame format is no longer supported in mlxtend >= 0.17.2. The allowed values are either 0/1 or True/False. For example, Apple Bananas Beer Chicken Milk Rice 0 True False True True False True 1 True False True False False True 2 True False True False False False 3 True True False False False False 4 False False True True True True 5 False False True False True True 6 False False True False True False 7 True True False False False False min_support : float (default: 0.5) A float between 0 and 1 for minimum support of the itemsets returned. The support is computed as the fraction transactions_where_item(s)_occur / total_transactions. use_colnames : bool (default: False) If true, uses the DataFrames' column names in the returned DataFrame instead of column indices. max_len : int (default: None) Given the set of all maximal itemsets, return those that are less than max_len . If None (default) all possible itemsets lengths are evaluated. verbose : int (default: 0) Shows the stages of conditional tree generation. Returns pandas DataFrame with columns ['support', 'itemsets'] of all maximal itemsets that are >= min_support and < than max_len (if max_len is not None). Each itemset in the 'itemsets' column is of type frozenset , which is a Python built-in type that behaves similarly to sets except that it is immutable (For more info, see https://docs.python.org/3.6/library/stdtypes.html#frozenset). Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/frequent_patterns/fpmax/","title":"Fpmax"},{"location":"user_guide/frequent_patterns/fpmax/#fpmax-maximal-itemsets-via-the-fp-max-algorithm","text":"Function implementing FP-Max to extract maximal itemsets for association rule mining from mlxtend.frequent_patterns import fpmax","title":"fpmax: Maximal itemsets via the FP-Max algorithm"},{"location":"user_guide/frequent_patterns/fpmax/#overview","text":"The Apriori algorithm is among the first and most popular algorithms for frequent itemset generation (frequent itemsets are then used for association rule mining). However, the runtime of Apriori can be quite large, especially for datasets with a large number of unique items, as the runtime grows exponentially depending on the number of unique items. In contrast to Apriori, FP-Growth is a frequent pattern generation algorithm that inserts items into a pattern search tree, which allows it to have a linear increase in runtime with respect to the number of unique items or entries. FP-Max is a variant of FP-Growth, which focuses on obtaining maximal itemsets. An itemset X is said to maximal if X is frequent and there exists no frequent super-pattern containing X. In other words, a frequent pattern X cannot be sub-pattern of larger frequent pattern to qualify for the definition maximal itemset .","title":"Overview"},{"location":"user_guide/frequent_patterns/fpmax/#references","text":"[1] Grahne, G., & Zhu, J. (2003, November). Efficiently using prefix-trees in mining frequent itemsets. In FIMI (Vol. 90).","title":"References"},{"location":"user_guide/frequent_patterns/fpmax/#related","text":"FP-Growth Apriori","title":"Related"},{"location":"user_guide/frequent_patterns/fpmax/#example-1-maximal-itemsets","text":"The fpmax function expects data in a one-hot encoded pandas DataFrame. Suppose we have the following transaction data: dataset = [['Milk', 'Onion', 'Nutmeg', 'Kidney Beans', 'Eggs', 'Yogurt'], ['Dill', 'Onion', 'Nutmeg', 'Kidney Beans', 'Eggs', 'Yogurt'], ['Milk', 'Apple', 'Kidney Beans', 'Eggs'], ['Milk', 'Unicorn', 'Corn', 'Kidney Beans', 'Yogurt'], ['Corn', 'Onion', 'Onion', 'Kidney Beans', 'Ice cream', 'Eggs']] We can transform it into the right format via the TransactionEncoder as follows: import pandas as pd from mlxtend.preprocessing import TransactionEncoder te = TransactionEncoder() te_ary = te.fit(dataset).transform(dataset) df = pd.DataFrame(te_ary, columns=te.columns_) df .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } Apple Corn Dill Eggs Ice cream Kidney Beans Milk Nutmeg Onion Unicorn Yogurt 0 False False False True False True True True True False True 1 False False True True False True False True True False True 2 True False False True False True True False False False False 3 False True False False False True True False False True True 4 False True False True True True False False True False False Now, let us return the items and itemsets with at least 60% support: from mlxtend.frequent_patterns import fpmax fpmax(df, min_support=0.6) .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } support itemsets 0 0.6 (5, 6) 1 0.6 (8, 3, 5) 2 0.6 (10, 5) By default, fpmax returns the column indices of the items, which may be useful in downstream operations such as association rule mining. For better readability, we can set use_colnames=True to convert these integer values into the respective item names: fpmax(df, min_support=0.6, use_colnames=True) .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } support itemsets 0 0.6 (Kidney Beans, Milk) 1 0.6 (Onion, Eggs, Kidney Beans) 2 0.6 (Kidney Beans, Yogurt)","title":"Example 1 -- Maximal Itemsets"},{"location":"user_guide/frequent_patterns/fpmax/#more-examples","text":"Please note that since the fpmax function is a drop-in replacement for fpgrowth and apriori , it comes with the same set of function arguments and return arguments. Thus, for more examples, please see the apriori documentation.","title":"More Examples"},{"location":"user_guide/frequent_patterns/fpmax/#api","text":"fpmax(df, min_support=0.5, use_colnames=False, max_len=None, verbose=0) Get maximal frequent itemsets from a one-hot DataFrame Parameters df : pandas DataFrame pandas DataFrame the encoded format. Also supports DataFrames with sparse data; for more info, please see (https://pandas.pydata.org/pandas-docs/stable/ user_guide/sparse.html#sparse-data-structures) Please note that the old pandas SparseDataFrame format is no longer supported in mlxtend >= 0.17.2. The allowed values are either 0/1 or True/False. For example, Apple Bananas Beer Chicken Milk Rice 0 True False True True False True 1 True False True False False True 2 True False True False False False 3 True True False False False False 4 False False True True True True 5 False False True False True True 6 False False True False True False 7 True True False False False False min_support : float (default: 0.5) A float between 0 and 1 for minimum support of the itemsets returned. The support is computed as the fraction transactions_where_item(s)_occur / total_transactions. use_colnames : bool (default: False) If true, uses the DataFrames' column names in the returned DataFrame instead of column indices. max_len : int (default: None) Given the set of all maximal itemsets, return those that are less than max_len . If None (default) all possible itemsets lengths are evaluated. verbose : int (default: 0) Shows the stages of conditional tree generation. Returns pandas DataFrame with columns ['support', 'itemsets'] of all maximal itemsets that are >= min_support and < than max_len (if max_len is not None). Each itemset in the 'itemsets' column is of type frozenset , which is a Python built-in type that behaves similarly to sets except that it is immutable (For more info, see https://docs.python.org/3.6/library/stdtypes.html#frozenset). Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/frequent_patterns/fpmax/","title":"API"},{"location":"user_guide/frequent_patterns/hmine/","text":"hmine : Frequent itemsets via the H-mine algorithm Hmine function to extract frequent itemsets for association rule mining from mlxtend.frequent_patterns import hmine Overview H-mine [1] (memory-based hyperstructure mining of frequent patterns) is a data mining algorithm used for frequent itemset mining -- the process of finding frequently occurring patterns in large transactional datasets. H-mine is an improvement over the Apriori and FP-Growth algorithms, offering better performance in terms of time and space complexity. It achieves this by using the H-struct data structure and a more efficient search space traversal method. H-mine improves upon the FP-Growth algorithm by introducing a novel data structure called the H-struct. The H-struct is a hybrid data structure that combines the benefits of both horizontal and vertical data layouts, making it more efficient for frequent itemset mining. A distinct feature of this method is that it has very limited and precisely predictable space overhead and runs really fast in memory-based settings. Moreover, it can be scaled up to very large databases by database partitioning, and when the data set becomes dense, (conditional) FP-trees can be constructed dynamically as part of the mining process. References [1] Pei J, Han J, Lu H, Nishio S, Tang S and Yang D, \" H-Mine: Fast and space-preserving frequent pattern mining in large databases. \" IIE Transactions, Vol. 39, pp. 593\u2013605, 2007. Related FP-Growth FP-Max Apriori Example 1 -- Generating Frequent Itemsets The hmine function expects data in a one-hot encoded pandas DataFrame. Suppose we have the following transaction data: dataset = [['Milk', 'Onion', 'Nutmeg', 'Kidney Beans', 'Eggs', 'Yogurt'], ['Dill', 'Onion', 'Nutmeg', 'Kidney Beans', 'Eggs', 'Yogurt'], ['Milk', 'Apple', 'Kidney Beans', 'Eggs'], ['Milk', 'Unicorn', 'Corn', 'Kidney Beans', 'Yogurt'], ['Corn', 'Onion', 'Onion', 'Kidney Beans', 'Ice cream', 'Eggs']] We can transform it into the right format via the TransactionEncoder as follows: import pandas as pd from mlxtend.preprocessing import TransactionEncoder te = TransactionEncoder() te_ary = te.fit(dataset).transform(dataset) df = pd.DataFrame(te_ary, columns=te.columns_) df .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } Apple Corn Dill Eggs Ice cream Kidney Beans Milk Nutmeg Onion Unicorn Yogurt 0 False False False True False True True True True False True 1 False False True True False True False True True False True 2 True False False True False True True False False False False 3 False True False False False True True False False True True 4 False True False True True True False False True False False Now, let us return the items and itemsets with at least 60% support: from mlxtend.frequent_patterns import hmine hmine(df, min_support=0.6) .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } support itemsets 0 0.8 (3) 1 0.8 (3, 5) 2 0.6 (8, 3, 5) 3 0.6 (8, 3) 4 1.0 (5) 5 0.6 (5, 6) 6 0.6 (8, 5) 7 0.6 (10, 5) 8 0.6 (6) 9 0.6 (8) 10 0.6 (10) By default, hmine returns the column indices of the items, which may be useful in downstream operations such as association rule mining. For better readability, we can set use_colnames=True to convert these integer values into the respective item names: hmine(df, min_support=0.6, use_colnames=True) .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } support itemsets 0 0.8 (Eggs) 1 0.8 (Eggs, Kidney Beans) 2 0.6 (Eggs, Kidney Beans, Onion) 3 0.6 (Eggs, Onion) 4 1.0 (Kidney Beans) 5 0.6 (Milk, Kidney Beans) 6 0.6 (Kidney Beans, Onion) 7 0.6 (Yogurt, Kidney Beans) 8 0.6 (Milk) 9 0.6 (Onion) 10 0.6 (Yogurt) Example 2 -- H-Mine versus Apriori and FP-Growth Since the hmine algorithm is a memory-based algorithm, it can be magnitudes faster than the alternative Apriori algorithm for large datasets. However, it can be much slower than the FP-Growth algorithm. In the following example, we compare the performance of hmine with the apriori and fpgrowth algorithms on a small dataset. import pandas as pd from mlxtend.preprocessing import TransactionEncoder te = TransactionEncoder() te_ary = te.fit(dataset).transform(dataset) df = pd.DataFrame(te_ary, columns=te.columns_) from mlxtend.frequent_patterns import apriori %timeit -n 100 -r 10 apriori(df, min_support=0.6, use_colnames=True) 3.41 ms \u00b1 584 \u00b5s per loop (mean \u00b1 std. dev. of 10 runs, 100 loops each) %timeit -n 100 -r 10 apriori(df, min_support=0.6, use_colnames=True, low_memory=True) 3.36 ms \u00b1 404 \u00b5s per loop (mean \u00b1 std. dev. of 10 runs, 100 loops each) from mlxtend.frequent_patterns import fpgrowth %timeit -n 100 -r 10 fpgrowth(df, min_support=0.6, use_colnames=True) 1.18 ms \u00b1 76.7 \u00b5s per loop (mean \u00b1 std. dev. of 10 runs, 100 loops each) from mlxtend.frequent_patterns import hmine %timeit -n 100 -r 10 hmine(df, min_support=0.6, use_colnames=True) 1.44 ms \u00b1 94.8 \u00b5s per loop (mean \u00b1 std. dev. of 10 runs, 100 loops each) Example 3 -- Working with Sparse Representations To save memory, you may want to represent your transaction data in the sparse format. This is especially useful if you have lots of products and small transactions. oht_ary = te.fit(dataset).transform(dataset, sparse=True) sparse_df = pd.DataFrame.sparse.from_spmatrix(oht_ary, columns=te.columns_) sparse_df .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } Apple Corn Dill Eggs Ice cream Kidney Beans Milk Nutmeg Onion Unicorn Yogurt 0 0 0 0 1 0 True 1 1 1 0 1 1 0 0 1 1 0 True 0 1 1 0 1 2 1 0 0 1 0 True 1 0 0 0 0 3 0 1 0 0 0 True 1 0 0 1 1 4 0 1 0 1 1 True 0 0 1 0 0 hmine(sparse_df, min_support=0.6, use_colnames=True, verbose=1) 2 itemset(s) from the suffixes on item(s) (Eggs) 1 itemset(s) from the suffixes on item(s) (Eggs, Kidney Beans) 0 itemset(s) from the suffixes on item(s) (Eggs, Kidney Beans, Onion) 0 itemset(s) from the suffixes on item(s) (Eggs, Onion) 3 itemset(s) from the suffixes on item(s) (Kidney Beans) 0 itemset(s) from the suffixes on item(s) (Kidney Beans, Milk) 0 itemset(s) from the suffixes on item(s) (Kidney Beans, Onion) 0 itemset(s) from the suffixes on item(s) (Kidney Beans, Yogurt) 0 itemset(s) from the suffixes on item(s) (Milk) 0 itemset(s) from the suffixes on item(s) (Onion) 0 itemset(s) from the suffixes on item(s) (Yogurt) .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } support itemsets 0 0.8 (Eggs) 1 0.8 (Eggs, Kidney Beans) 2 0.6 (Eggs, Kidney Beans, Onion) 3 0.6 (Eggs, Onion) 4 1.0 (Kidney Beans) 5 0.6 (Milk, Kidney Beans) 6 0.6 (Kidney Beans, Onion) 7 0.6 (Yogurt, Kidney Beans) 8 0.6 (Milk) 9 0.6 (Onion) 10 0.6 (Yogurt) More Examples Please note that since the hmine function is a drop-in replacement for apriori and fpgrowth , it comes with the same set of function arguments and return arguments. Thus, for more examples, please see the apriori documentation. API hmine(df, min_support=0.5, use_colnames=False, max_len=None, verbose=0) -> pandas.core.frame.DataFrame Get frequent itemsets from a one-hot DataFrame Parameters df : pandas DataFrame pandas DataFrame the encoded format. Also supports DataFrames with sparse data; for more info, please see https://pandas.pydata.org/pandas-docs/stable/user_guide/sparse.html#sparse-data-structures. Please note that the old pandas SparseDataFrame format is no longer supported in mlxtend >= 0.17.2. The allowed values are either 0/1 or True/False. For example, Apple Bananas Beer Chicken Milk Rice 0 True False True True False True 1 True False True False False True 2 True False True False False False 3 True True False False False False 4 False False True True True True 5 False False True False True True 6 False False True False True False 7 True True False False False False min_support : float (default: 0.5) A float between 0 and 1 for minimum support of the itemsets returned. The support is computed as the fraction transactions_where_item(s)_occur / total_transactions. use_colnames : bool (default: False) If true, uses the DataFrames' column names in the returned DataFrame instead of column indices. max_len : int (default: None) Maximum length of the itemsets generated. If None (default) all possible itemsets lengths are evaluated. verbose : int (default: 0) Shows the stages of conditional tree generation. Returns pandas DataFrame with columns ['support', 'itemsets'] of all itemsets that are >= min_support and < than max_len (if max_len is not None). Each itemset in the 'itemsets' column is of type frozenset , which is a Python built-in type that behaves similarly to sets except that it is immutable (For more info, see https://docs.python.org/3.6/library/stdtypes.html#frozenset). Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/frequent_patterns/hmine/ ython","title":"Hmine"},{"location":"user_guide/frequent_patterns/hmine/#hmine-frequent-itemsets-via-the-h-mine-algorithm","text":"Hmine function to extract frequent itemsets for association rule mining from mlxtend.frequent_patterns import hmine","title":"hmine: Frequent itemsets via the H-mine algorithm"},{"location":"user_guide/frequent_patterns/hmine/#overview","text":"H-mine [1] (memory-based hyperstructure mining of frequent patterns) is a data mining algorithm used for frequent itemset mining -- the process of finding frequently occurring patterns in large transactional datasets. H-mine is an improvement over the Apriori and FP-Growth algorithms, offering better performance in terms of time and space complexity. It achieves this by using the H-struct data structure and a more efficient search space traversal method. H-mine improves upon the FP-Growth algorithm by introducing a novel data structure called the H-struct. The H-struct is a hybrid data structure that combines the benefits of both horizontal and vertical data layouts, making it more efficient for frequent itemset mining. A distinct feature of this method is that it has very limited and precisely predictable space overhead and runs really fast in memory-based settings. Moreover, it can be scaled up to very large databases by database partitioning, and when the data set becomes dense, (conditional) FP-trees can be constructed dynamically as part of the mining process.","title":"Overview"},{"location":"user_guide/frequent_patterns/hmine/#references","text":"[1] Pei J, Han J, Lu H, Nishio S, Tang S and Yang D, \" H-Mine: Fast and space-preserving frequent pattern mining in large databases. \" IIE Transactions, Vol. 39, pp. 593\u2013605, 2007.","title":"References"},{"location":"user_guide/frequent_patterns/hmine/#related","text":"FP-Growth FP-Max Apriori","title":"Related"},{"location":"user_guide/frequent_patterns/hmine/#example-1-generating-frequent-itemsets","text":"The hmine function expects data in a one-hot encoded pandas DataFrame. Suppose we have the following transaction data: dataset = [['Milk', 'Onion', 'Nutmeg', 'Kidney Beans', 'Eggs', 'Yogurt'], ['Dill', 'Onion', 'Nutmeg', 'Kidney Beans', 'Eggs', 'Yogurt'], ['Milk', 'Apple', 'Kidney Beans', 'Eggs'], ['Milk', 'Unicorn', 'Corn', 'Kidney Beans', 'Yogurt'], ['Corn', 'Onion', 'Onion', 'Kidney Beans', 'Ice cream', 'Eggs']] We can transform it into the right format via the TransactionEncoder as follows: import pandas as pd from mlxtend.preprocessing import TransactionEncoder te = TransactionEncoder() te_ary = te.fit(dataset).transform(dataset) df = pd.DataFrame(te_ary, columns=te.columns_) df .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } Apple Corn Dill Eggs Ice cream Kidney Beans Milk Nutmeg Onion Unicorn Yogurt 0 False False False True False True True True True False True 1 False False True True False True False True True False True 2 True False False True False True True False False False False 3 False True False False False True True False False True True 4 False True False True True True False False True False False Now, let us return the items and itemsets with at least 60% support: from mlxtend.frequent_patterns import hmine hmine(df, min_support=0.6) .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } support itemsets 0 0.8 (3) 1 0.8 (3, 5) 2 0.6 (8, 3, 5) 3 0.6 (8, 3) 4 1.0 (5) 5 0.6 (5, 6) 6 0.6 (8, 5) 7 0.6 (10, 5) 8 0.6 (6) 9 0.6 (8) 10 0.6 (10) By default, hmine returns the column indices of the items, which may be useful in downstream operations such as association rule mining. For better readability, we can set use_colnames=True to convert these integer values into the respective item names: hmine(df, min_support=0.6, use_colnames=True) .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } support itemsets 0 0.8 (Eggs) 1 0.8 (Eggs, Kidney Beans) 2 0.6 (Eggs, Kidney Beans, Onion) 3 0.6 (Eggs, Onion) 4 1.0 (Kidney Beans) 5 0.6 (Milk, Kidney Beans) 6 0.6 (Kidney Beans, Onion) 7 0.6 (Yogurt, Kidney Beans) 8 0.6 (Milk) 9 0.6 (Onion) 10 0.6 (Yogurt)","title":"Example 1 -- Generating Frequent Itemsets"},{"location":"user_guide/frequent_patterns/hmine/#example-2-h-mine-versus-apriori-and-fp-growth","text":"Since the hmine algorithm is a memory-based algorithm, it can be magnitudes faster than the alternative Apriori algorithm for large datasets. However, it can be much slower than the FP-Growth algorithm. In the following example, we compare the performance of hmine with the apriori and fpgrowth algorithms on a small dataset. import pandas as pd from mlxtend.preprocessing import TransactionEncoder te = TransactionEncoder() te_ary = te.fit(dataset).transform(dataset) df = pd.DataFrame(te_ary, columns=te.columns_) from mlxtend.frequent_patterns import apriori %timeit -n 100 -r 10 apriori(df, min_support=0.6, use_colnames=True) 3.41 ms \u00b1 584 \u00b5s per loop (mean \u00b1 std. dev. of 10 runs, 100 loops each) %timeit -n 100 -r 10 apriori(df, min_support=0.6, use_colnames=True, low_memory=True) 3.36 ms \u00b1 404 \u00b5s per loop (mean \u00b1 std. dev. of 10 runs, 100 loops each) from mlxtend.frequent_patterns import fpgrowth %timeit -n 100 -r 10 fpgrowth(df, min_support=0.6, use_colnames=True) 1.18 ms \u00b1 76.7 \u00b5s per loop (mean \u00b1 std. dev. of 10 runs, 100 loops each) from mlxtend.frequent_patterns import hmine %timeit -n 100 -r 10 hmine(df, min_support=0.6, use_colnames=True) 1.44 ms \u00b1 94.8 \u00b5s per loop (mean \u00b1 std. dev. of 10 runs, 100 loops each)","title":"Example 2 -- H-Mine versus Apriori and FP-Growth"},{"location":"user_guide/frequent_patterns/hmine/#example-3-working-with-sparse-representations","text":"To save memory, you may want to represent your transaction data in the sparse format. This is especially useful if you have lots of products and small transactions. oht_ary = te.fit(dataset).transform(dataset, sparse=True) sparse_df = pd.DataFrame.sparse.from_spmatrix(oht_ary, columns=te.columns_) sparse_df .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } Apple Corn Dill Eggs Ice cream Kidney Beans Milk Nutmeg Onion Unicorn Yogurt 0 0 0 0 1 0 True 1 1 1 0 1 1 0 0 1 1 0 True 0 1 1 0 1 2 1 0 0 1 0 True 1 0 0 0 0 3 0 1 0 0 0 True 1 0 0 1 1 4 0 1 0 1 1 True 0 0 1 0 0 hmine(sparse_df, min_support=0.6, use_colnames=True, verbose=1) 2 itemset(s) from the suffixes on item(s) (Eggs) 1 itemset(s) from the suffixes on item(s) (Eggs, Kidney Beans) 0 itemset(s) from the suffixes on item(s) (Eggs, Kidney Beans, Onion) 0 itemset(s) from the suffixes on item(s) (Eggs, Onion) 3 itemset(s) from the suffixes on item(s) (Kidney Beans) 0 itemset(s) from the suffixes on item(s) (Kidney Beans, Milk) 0 itemset(s) from the suffixes on item(s) (Kidney Beans, Onion) 0 itemset(s) from the suffixes on item(s) (Kidney Beans, Yogurt) 0 itemset(s) from the suffixes on item(s) (Milk) 0 itemset(s) from the suffixes on item(s) (Onion) 0 itemset(s) from the suffixes on item(s) (Yogurt) .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } support itemsets 0 0.8 (Eggs) 1 0.8 (Eggs, Kidney Beans) 2 0.6 (Eggs, Kidney Beans, Onion) 3 0.6 (Eggs, Onion) 4 1.0 (Kidney Beans) 5 0.6 (Milk, Kidney Beans) 6 0.6 (Kidney Beans, Onion) 7 0.6 (Yogurt, Kidney Beans) 8 0.6 (Milk) 9 0.6 (Onion) 10 0.6 (Yogurt)","title":"Example 3 -- Working with Sparse Representations"},{"location":"user_guide/frequent_patterns/hmine/#more-examples","text":"Please note that since the hmine function is a drop-in replacement for apriori and fpgrowth , it comes with the same set of function arguments and return arguments. Thus, for more examples, please see the apriori documentation.","title":"More Examples"},{"location":"user_guide/frequent_patterns/hmine/#api","text":"hmine(df, min_support=0.5, use_colnames=False, max_len=None, verbose=0) -> pandas.core.frame.DataFrame Get frequent itemsets from a one-hot DataFrame Parameters df : pandas DataFrame pandas DataFrame the encoded format. Also supports DataFrames with sparse data; for more info, please see https://pandas.pydata.org/pandas-docs/stable/user_guide/sparse.html#sparse-data-structures. Please note that the old pandas SparseDataFrame format is no longer supported in mlxtend >= 0.17.2. The allowed values are either 0/1 or True/False. For example, Apple Bananas Beer Chicken Milk Rice 0 True False True True False True 1 True False True False False True 2 True False True False False False 3 True True False False False False 4 False False True True True True 5 False False True False True True 6 False False True False True False 7 True True False False False False min_support : float (default: 0.5) A float between 0 and 1 for minimum support of the itemsets returned. The support is computed as the fraction transactions_where_item(s)_occur / total_transactions. use_colnames : bool (default: False) If true, uses the DataFrames' column names in the returned DataFrame instead of column indices. max_len : int (default: None) Maximum length of the itemsets generated. If None (default) all possible itemsets lengths are evaluated. verbose : int (default: 0) Shows the stages of conditional tree generation. Returns pandas DataFrame with columns ['support', 'itemsets'] of all itemsets that are >= min_support and < than max_len (if max_len is not None). Each itemset in the 'itemsets' column is of type frozenset , which is a Python built-in type that behaves similarly to sets except that it is immutable (For more info, see https://docs.python.org/3.6/library/stdtypes.html#frozenset). Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/frequent_patterns/hmine/ ython","title":"API"},{"location":"user_guide/image/extract_face_landmarks/","text":"extract_face_landmarks: extract 68 landmark features from face images A function extract facial landmarks. from mlxtend.image import extract_face_landmarks Overview The extract_face_landmarks function detects the faces in a given image, and then it will return the face landmark points (also known as face shape) for the first found face in the image based on dlib's face landmark detection code (https://dlib.net/face_landmark_detection_ex.cpp.html): The face detector we use is made using the classic Histogram of Oriented Gradients (HOG) feature combined with a linear classifier, an image pyramid, and sliding window detection scheme. The pose estimator was created by using dlib's implementation of the paper: One Millisecond Face Alignment with an Ensemble of Regression Trees by Vahid Kazemi and Josephine Sullivan, CVPR 2014 and was trained on the iBUG 300-W face landmark dataset (see https://ibug.doc.ic.ac.uk/resources/facial-point-annotations/): C. Sagonas, E. Antonakos, G, Tzimiropoulos, S. Zafeiriou, M. Pantic. 300 faces In-the-wild challenge: Database and results. Image and Vision Computing (IMAVIS), Special Issue on Facial Landmark Localisation \"In-The-Wild\". 2016. You can get the trained model file from: https://dlib.net/files/shape_predictor_68_face_landmarks.dat.bz2. Note that the license for the iBUG 300-W dataset excludes commercial use. So you should contact Imperial College London to find out if it's OK for you to use this model file in a commercial product. References Kazemi, Vahid, and Josephine Sullivan. \"One millisecond face alignment with an ensemble of regression trees.\" Proceedings of the IEEE Conference on Computer Vision and Pattern Recognition. 2014. Example 1 -- RGB Images import imageio import matplotlib.pyplot as plt from mlxtend.image import extract_face_landmarks img = imageio.imread('test-face.png') landmarks = extract_face_landmarks(img) print(landmarks.shape) print('\\n\\nFirst 10 landmarks:\\n', landmarks[:10]) (68, 2) First 10 landmarks: [[132 159] [134 173] [137 187] [142 200] [150 212] [159 222] [170 231] [184 235] [198 236] [211 231]] Visualization of the landmarks: fig = plt.figure(figsize=(15, 5)) ax = fig.add_subplot(1, 3, 1) ax.imshow(img) ax = fig.add_subplot(1, 3, 2) ax.scatter(landmarks[:, 0], -landmarks[:, 1], alpha=0.8) ax = fig.add_subplot(1, 3, 3) img2 = img.copy() for p in landmarks: img2[p[1]-3:p[1]+3, p[0]-3:p[0]+3, :] = (255, 255, 255) # note that the values -3 and +3 will make the landmarks # overlayed on the image 6 pixels wide; depending on the # resolution of the face image, you may want to change # this value ax.imshow(img2) plt.show() Displaying the index of landmark points import numpy as np left_eye = np.array([36, 37, 38, 39, 40, 41]) right_eye = np.array([42, 43, 44, 45, 46, 47]) import matplotlib.pyplot as plt %matplotlib inline fig = plt.figure(figsize=(10,10)) plt.plot(landmarks[:,0], -landmarks[:,1], 'ro', markersize=8, alpha = 0.5) for i in range(landmarks.shape[0]): plt.text(landmarks[i,0]+1, -landmarks[i,1], str(i), size=14) left_eye_center = np.mean(landmarks[left_eye], axis=0) right_eye_center = np.mean(landmarks[right_eye], axis=0) print('Coordinates of the Left Eye: ', left_eye_center) print('Coordinates of the Right Eye: ', right_eye_center) plt.plot([left_eye_center[0]], [-left_eye_center[1]], marker='+', color='blue', markersize=10, mew=4) plt.plot([right_eye_center[0]], [-right_eye_center[1]], marker='+', color='blue', markersize=10, mew=4) plt.xticks([]) plt.yticks([]) plt.show() Coordinates of the Left Eye: [169.33333333 156. ] Coordinates of the Right Eye: [210.83333333 152.16666667] Example 2 -- Grayscale Images import imageio import matplotlib.pyplot as plt from mlxtend.image import extract_face_landmarks img = imageio.imread('test-face_grayscale_lowres.png', ) landmarks = extract_face_landmarks(img) print(landmarks.shape) print('\\n\\nFirst 10 landmarks:\\n', landmarks[:10]) (68, 2) First 10 landmarks: [[37 42] [38 46] [39 51] [41 55] [44 59] [47 62] [51 65] [55 66] [60 66] [64 64]] fig = plt.figure(figsize=(15, 5)) ax = fig.add_subplot(1, 3, 1) ax.imshow(img, cmap='gray') ax = fig.add_subplot(1, 3, 2) ax.scatter(landmarks[:, 0], -landmarks[:, 1], alpha=0.8) ax = fig.add_subplot(1, 3, 3) img2 = img.copy() for p in landmarks: img2[p[1]-1:p[1]+1, p[0]-1:p[0]+1] = 255 # note that the values -1 and +1 will make the landmarks # overlayed on the image 2 pixels wide; depending on the # resolution of the face image, you may want to change # this value ax.imshow(img2, cmap='gray') plt.show() API extract_face_landmarks(img, return_dtype= ) Function to extract face landmarks. Note that this function requires an installation of the Python version of the library \"dlib\": https://dlib.net Parameters img : array, shape = [h, w, ?] Numpy array of a face image or imageio.core.util.Array. E.g., img = imageio.core.util.Array(ary) Supported shapes are - 3D tensors with 1 or more color channels, for example, RGB: [h, w, 3] - 2D tensors without color channel, for example, Grayscale: [h, w] return_dtype: the return data-type of the array, default: np.int32. Returns landmarks : numpy.ndarray, shape = [68, 2] A numpy array, where each row contains a landmark/point x-y coordinates. Return None if no face is detected by Dlib. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/image/extract_face_landmarks/","title":"extract_face_landmarks: extract 68 landmark features from face images"},{"location":"user_guide/image/extract_face_landmarks/#extract_face_landmarks-extract-68-landmark-features-from-face-images","text":"A function extract facial landmarks. from mlxtend.image import extract_face_landmarks","title":"extract_face_landmarks: extract 68 landmark features from face images"},{"location":"user_guide/image/extract_face_landmarks/#overview","text":"The extract_face_landmarks function detects the faces in a given image, and then it will return the face landmark points (also known as face shape) for the first found face in the image based on dlib's face landmark detection code (https://dlib.net/face_landmark_detection_ex.cpp.html): The face detector we use is made using the classic Histogram of Oriented Gradients (HOG) feature combined with a linear classifier, an image pyramid, and sliding window detection scheme. The pose estimator was created by using dlib's implementation of the paper: One Millisecond Face Alignment with an Ensemble of Regression Trees by Vahid Kazemi and Josephine Sullivan, CVPR 2014 and was trained on the iBUG 300-W face landmark dataset (see https://ibug.doc.ic.ac.uk/resources/facial-point-annotations/): C. Sagonas, E. Antonakos, G, Tzimiropoulos, S. Zafeiriou, M. Pantic. 300 faces In-the-wild challenge: Database and results. Image and Vision Computing (IMAVIS), Special Issue on Facial Landmark Localisation \"In-The-Wild\". 2016. You can get the trained model file from: https://dlib.net/files/shape_predictor_68_face_landmarks.dat.bz2. Note that the license for the iBUG 300-W dataset excludes commercial use. So you should contact Imperial College London to find out if it's OK for you to use this model file in a commercial product.","title":"Overview"},{"location":"user_guide/image/extract_face_landmarks/#references","text":"Kazemi, Vahid, and Josephine Sullivan. \"One millisecond face alignment with an ensemble of regression trees.\" Proceedings of the IEEE Conference on Computer Vision and Pattern Recognition. 2014.","title":"References"},{"location":"user_guide/image/extract_face_landmarks/#example-1-rgb-images","text":"import imageio import matplotlib.pyplot as plt from mlxtend.image import extract_face_landmarks img = imageio.imread('test-face.png') landmarks = extract_face_landmarks(img) print(landmarks.shape) print('\\n\\nFirst 10 landmarks:\\n', landmarks[:10]) (68, 2) First 10 landmarks: [[132 159] [134 173] [137 187] [142 200] [150 212] [159 222] [170 231] [184 235] [198 236] [211 231]] Visualization of the landmarks: fig = plt.figure(figsize=(15, 5)) ax = fig.add_subplot(1, 3, 1) ax.imshow(img) ax = fig.add_subplot(1, 3, 2) ax.scatter(landmarks[:, 0], -landmarks[:, 1], alpha=0.8) ax = fig.add_subplot(1, 3, 3) img2 = img.copy() for p in landmarks: img2[p[1]-3:p[1]+3, p[0]-3:p[0]+3, :] = (255, 255, 255) # note that the values -3 and +3 will make the landmarks # overlayed on the image 6 pixels wide; depending on the # resolution of the face image, you may want to change # this value ax.imshow(img2) plt.show()","title":"Example 1 -- RGB Images"},{"location":"user_guide/image/extract_face_landmarks/#displaying-the-index-of-landmark-points","text":"import numpy as np left_eye = np.array([36, 37, 38, 39, 40, 41]) right_eye = np.array([42, 43, 44, 45, 46, 47]) import matplotlib.pyplot as plt %matplotlib inline fig = plt.figure(figsize=(10,10)) plt.plot(landmarks[:,0], -landmarks[:,1], 'ro', markersize=8, alpha = 0.5) for i in range(landmarks.shape[0]): plt.text(landmarks[i,0]+1, -landmarks[i,1], str(i), size=14) left_eye_center = np.mean(landmarks[left_eye], axis=0) right_eye_center = np.mean(landmarks[right_eye], axis=0) print('Coordinates of the Left Eye: ', left_eye_center) print('Coordinates of the Right Eye: ', right_eye_center) plt.plot([left_eye_center[0]], [-left_eye_center[1]], marker='+', color='blue', markersize=10, mew=4) plt.plot([right_eye_center[0]], [-right_eye_center[1]], marker='+', color='blue', markersize=10, mew=4) plt.xticks([]) plt.yticks([]) plt.show() Coordinates of the Left Eye: [169.33333333 156. ] Coordinates of the Right Eye: [210.83333333 152.16666667]","title":"Displaying the index of landmark points"},{"location":"user_guide/image/extract_face_landmarks/#example-2-grayscale-images","text":"import imageio import matplotlib.pyplot as plt from mlxtend.image import extract_face_landmarks img = imageio.imread('test-face_grayscale_lowres.png', ) landmarks = extract_face_landmarks(img) print(landmarks.shape) print('\\n\\nFirst 10 landmarks:\\n', landmarks[:10]) (68, 2) First 10 landmarks: [[37 42] [38 46] [39 51] [41 55] [44 59] [47 62] [51 65] [55 66] [60 66] [64 64]] fig = plt.figure(figsize=(15, 5)) ax = fig.add_subplot(1, 3, 1) ax.imshow(img, cmap='gray') ax = fig.add_subplot(1, 3, 2) ax.scatter(landmarks[:, 0], -landmarks[:, 1], alpha=0.8) ax = fig.add_subplot(1, 3, 3) img2 = img.copy() for p in landmarks: img2[p[1]-1:p[1]+1, p[0]-1:p[0]+1] = 255 # note that the values -1 and +1 will make the landmarks # overlayed on the image 2 pixels wide; depending on the # resolution of the face image, you may want to change # this value ax.imshow(img2, cmap='gray') plt.show()","title":"Example 2 -- Grayscale Images"},{"location":"user_guide/image/extract_face_landmarks/#api","text":"extract_face_landmarks(img, return_dtype= ) Function to extract face landmarks. Note that this function requires an installation of the Python version of the library \"dlib\": https://dlib.net Parameters img : array, shape = [h, w, ?] Numpy array of a face image or imageio.core.util.Array. E.g., img = imageio.core.util.Array(ary) Supported shapes are - 3D tensors with 1 or more color channels, for example, RGB: [h, w, 3] - 2D tensors without color channel, for example, Grayscale: [h, w] return_dtype: the return data-type of the array, default: np.int32. Returns landmarks : numpy.ndarray, shape = [68, 2] A numpy array, where each row contains a landmark/point x-y coordinates. Return None if no face is detected by Dlib. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/image/extract_face_landmarks/","title":"API"},{"location":"user_guide/image/eyepad_align/","text":"EyepadAlign: align face images based on eye location A class to align face images based on eye location. from mlxtend.image import EyepadAlign Overview The EyepadAlign class align face images to target face landmarks based on the location of the eyes. The facial landmarks are detected using mlxted.image.extract_face_landmarks , which is based on dlib's face landmark detection code [1]. EyepadAlign implements 3 fit methods, fit_image , fit_values , and fit_directory ; once the EyepadAlign object has been fitted via one of these methods (as illustrated in the examples), it can be used to transform a target image. References [1] Kazemi, Vahid, and Josephine Sullivan. \"One millisecond face alignment with an ensemble of regression trees.\" Proceedings of the IEEE Conference on Computer Vision and Pattern Recognition. 2014. Example 1 (fit_image) This example illustrates the use of EyepadAlign fit to a single target input image (first image in the CelebA dataset ) to another portrait such that the eye position is in the same scale and location as the target image. import imageio from mlxtend.image import EyepadAlign ### Fit EyepadAlign on a single target image eyepad = EyepadAlign() target_image = imageio.imread('celeba-subset/000001.jpg') print('Target image shape: ', target_image.shape) eyepad.fit_image(target_image); Target image shape: (218, 178, 3) ### Transform a test image img = imageio.imread('test-face.png') print('Original Image shape: ', img.shape) img_tr = eyepad.transform(img) print('Transformed Image shape: ', img_tr.shape) Original Image shape: (350, 350, 3) Transformed Image shape: (218, 178, 3) import matplotlib.pyplot as plt fig = plt.figure(figsize=(12, 5)) ax = fig.add_subplot(1, 3, 1) ax.imshow(target_image) ax.set_title('Target', size=20) ax = fig.add_subplot(1, 3, 2) ax.imshow(img) ax.set_title('Original', size=20) ax = fig.add_subplot(1, 3, 3) ax.imshow(img_tr) ax.set_title('Aligned to Target', size=20) plt.show() Example 2 (fit_directory) In typical computer vision and deep learning tasks, we are interested in aligning all images in a dataset and across dataset to allow cross-dataset comparisons of face image classifiers. Using the fit_directory method, we can fit EyepadAlign such that the average landmarks are computed from all images in a directory. Then, we can use these average landmarks to transform new images based on the average eye location. import imageio from mlxtend.image import EyepadAlign ### Setting verbosity to 1 will print a progress bar eyepad = EyepadAlign(verbose=1) eyepad.fit_directory(target_img_dir='celeba-subset/', target_width=178, target_height=218, file_extension='.jpg') img = imageio.imread('test-face.png') img_tr = eyepad.transform(img) print('Transformed Image shape: ', img_tr.shape) Pre-Checking directory for consistent image dimensions... 0% [#########] 100% | ETA: 00:00:00 Fitting the average facial landmarks for 9 face images Total time elapsed: 00:00:00 0% [##### ] 100% | ETA: 00:00:00/Users/sebastian/code/mlxtend/mlxtend/image/extract_face_landmarks.py:66: UserWarning: No face detected. warnings.warn('No face detected.') /Users/sebastian/code/mlxtend/mlxtend/image/eyepad_align.py:188: UserWarning: No face detected in image 000004.jpg. Image ignored. % f) 0% [#########] 100% | ETA: 00:00:00 Total time elapsed: 00:00:00 /Users/sebastian/code/mlxtend/mlxtend/image/eyepad_align.py:188: UserWarning: No face detected in image 000003.jpg. Image ignored. % f) Transformed Image shape: (218, 178, 3) Note that a warning is displayed if no face landmarks could be detected. In these cases, the image is skipped and not included in the average landmark computation. E.g., 000003.jpg would be such an example: import matplotlib.pyplot as plt img_no_landmarks = imageio.imread('celeba-subset/000003.jpg') plt.imshow(img_no_landmarks) plt.show() Finally, the an example of aligning the original face image to the average landmarks in the dataset is shown below: fig = plt.figure(figsize=(10, 4)) ax = fig.add_subplot(1, 3, 1) ax.scatter(eyepad.target_landmarks_[:, 0], eyepad.target_landmarks_[:, 1], alpha=0.8) ax.set_title('Target Landmarks', size=20) ax.set_xlim(0, 178) ax.set_ylim(218, 0) ax = fig.add_subplot(1, 3, 2) ax.imshow(img) ax.set_title('Original', size=20) ax = fig.add_subplot(1, 3, 3) ax.imshow(img_tr) ax.set_title('Aligned to Target', size=20) plt.tight_layout() plt.show() Example 3 (fit_values) For more customization, landmarks can also be passed manually. import imageio from mlxtend.image import EyepadAlign import numpy as np celeba_landmarks = np.array([[45, 113], [45, 124], [46, 136], [49, 147], [53, 158], [60, 167], [68, 175], [77, 182], [88, 184], [99, 182], [108, 175], [117, 167], [124, 158], [128, 147], [131, 136], [132, 124], [132, 113], [53, 104], [58, 99], [66, 98], [73, 99], [80, 102], [95, 102], [103, 99], [110, 98], [118, 99], [123, 104], [88, 111], [88, 119], [88, 126], [88, 134], [79, 139], [83, 140], [88, 142], [92, 140], [96, 139], [62, 112], [66, 109], [72, 109], [77, 113], [71, 114], [66, 114], [99, 113], [104, 109], [110, 109], [115, 112], [110, 114], [105, 114], [71, 152], [77, 150], [83, 149], [88, 150], [92, 149], [99, 150], [105, 152], [99, 159], [93, 162], [88, 163], [83, 162], [77, 159], [73, 153], [83, 152], [88, 153], [93, 152], [103, 153], [93, 156], [88, 157], [83, 156]]) eyepad = EyepadAlign(verbose=1) eyepad.fit_values(target_landmarks=celeba_landmarks, target_width=178, target_height=218); import matplotlib.pyplot as plt img = imageio.imread('test-face.png') img_tr = eyepad.transform(img) print('Transformed Image shape: ', img_tr.shape) fig = plt.figure(figsize=(10, 4)) ax = fig.add_subplot(1, 3, 1) ax.scatter(eyepad.target_landmarks_[:, 0], eyepad.target_landmarks_[:, 1], alpha=0.8) ax.set_title('Target Landmarks', size=20) ax.set_xlim(0, 178) ax.set_ylim(218, 0) ax = fig.add_subplot(1, 3, 2) ax.imshow(img) ax.set_title('Original', size=20) ax = fig.add_subplot(1, 3, 3) ax.imshow(img_tr) ax.set_title('Aligned to Target', size=20) plt.tight_layout() plt.show() Transformed Image shape: (218, 178, 3) Example 4 (centering the nose) In many applications that involve center-cropping, it is helpful to translate the image such that the tip of the nose is located in the center of the image. ### Fit EyepadAlign to dataset to compute average landmarks import imageio from mlxtend.image import EyepadAlign import numpy as np eyepad = EyepadAlign(verbose=1) eyepad.fit_directory(target_img_dir='celeba-subset/', target_width=178, target_height=218, file_extension='.jpg') Pre-Checking directory for consistent image dimensions... 0% [#########] 100% | ETA: 00:00:00 Fitting the average facial landmarks for 9 face images Total time elapsed: 00:00:00 0% [#########] 100% | ETA: 00:00:00 Total time elapsed: 00:00:00 <mlxtend.image.eyepad_align.EyepadAlign at 0x7fa09865d390> ### Use the calculated landmarks to center the nose: center = np.array([178//2, 218//2]) displacement_vector = center - eyepad.target_landmarks_[33] # index 33 is the nose-tip nose_centered_landmarks = eyepad.target_landmarks_ + displacement_vector eyepad_cent_nose = EyepadAlign(verbose=1) eyepad_cent_nose.fit_values(target_landmarks=nose_centered_landmarks, target_width=178, target_height=218); ### Evaluate the results img = imageio.imread('test-face.png') img_tr = eyepad.transform(img) img_nose_centered = eyepad_cent_nose.transform(img) fig = plt.figure(figsize=(10, 4)) ax = fig.add_subplot(1, 3, 1) ax.imshow(img) ax.set_title('Original', size=20) ax = fig.add_subplot(1, 3, 2) ax.imshow(img_tr) ax.set_title('Aligned (Average eye \\n position in dataset)', size=15) ax = fig.add_subplot(1, 3, 3) ax.imshow(img_nose_centered) ax.set_title('Aligned (Nose in image center)', size=15) plt.tight_layout() plt.show() API EyepadAlign(verbose=0) Class to align/transform face images to facial landmarks, based on eye alignment. A scaling factor is computed based on distance between the left and right eye, such that the transformed face image will have the same eye distance as a reference face image. A transformation is performed based on the eyes' center point. to align the face based on the reference eye location. Finally, the transformed image is padded with zeros to match the desired final image size. Parameters verbose : int (default=0) Verbose level to display the progress bar and log messages. Setting verbose=1 will print a progress bar upon calling fit_directory . Attributes target_landmarks_ : target landmarks to transform new face images to. Depending on the chosen fit parameters, it can be either (1) assigned to pre-fit shapes, (2) computed from a single face image (3) computed as the mean of face landmarks from all face images in a file directory of face images. eye_distance_ : the distance between left and right eyes in the target landmarks. target_height_ : the height of the transformed output image. target_width_ : the width of the transformed output image. Returns self : object Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/image/eyepad_align/ Methods fit_directory(target_img_dir, target_height, target_width, file_extension='.jpg', pre_check=True) Calculates the average landmarks for all face images in a directory which will then be set as the target landmark set. Arguments target_img_dir : str Directory containing the images target_height : int Expected image height of the images in the directory target_width : int Expected image width of the images in the directory file_extension str (default='.jpg'): File extension of the image files. pre_check Bool (default=True): Checks that each image has the dimensions specificed via target_height and target_width on the whole directory first to identify potential issues that are recommended to be fixed before proceeding. Raises a warning for each image if dimensions differ from the ones specified and expected. Returns self : object fit_image(target_image) Derives facial landmarks from a target image. Arguments target_image : uint8 numpy.array, shape=[width, height, channels] NumPy array representation of the image data. Returns self : object fit_values(target_landmarks, target_width, target_height) Used for determining the eye location from pre-defined landmark arrays, eliminating the need for re-computing the average landmarks on a target image or image directory. Arguments target_landmarks : np.array, shape=(height, width) NumPy array containing the locations of the facial landmarks as determined by mlxtend.image.extract_face_landmarks target_height : int image height target_width : int image width Returns self : object transform(img) transforms a single face image (img) to the target landmarks based on the location of the eyes by scaling, translation and cropping (if needed): (1) Scaling the image so that the distance of the two eyes in the given image (img) matches the distance of the two eyes in the target landmarks. (2) Translation is performed based on the middle point between the two eyes. Arguments img : np.array, shape=(height, width, channels) Input image to be transformed. Returns self : object","title":"EyepadAlign:  align face images based on eye location"},{"location":"user_guide/image/eyepad_align/#eyepadalign-align-face-images-based-on-eye-location","text":"A class to align face images based on eye location. from mlxtend.image import EyepadAlign","title":"EyepadAlign:  align face images based on eye location"},{"location":"user_guide/image/eyepad_align/#overview","text":"The EyepadAlign class align face images to target face landmarks based on the location of the eyes. The facial landmarks are detected using mlxted.image.extract_face_landmarks , which is based on dlib's face landmark detection code [1]. EyepadAlign implements 3 fit methods, fit_image , fit_values , and fit_directory ; once the EyepadAlign object has been fitted via one of these methods (as illustrated in the examples), it can be used to transform a target image.","title":"Overview"},{"location":"user_guide/image/eyepad_align/#references","text":"[1] Kazemi, Vahid, and Josephine Sullivan. \"One millisecond face alignment with an ensemble of regression trees.\" Proceedings of the IEEE Conference on Computer Vision and Pattern Recognition. 2014.","title":"References"},{"location":"user_guide/image/eyepad_align/#example-1-fit_image","text":"This example illustrates the use of EyepadAlign fit to a single target input image (first image in the CelebA dataset ) to another portrait such that the eye position is in the same scale and location as the target image. import imageio from mlxtend.image import EyepadAlign ### Fit EyepadAlign on a single target image eyepad = EyepadAlign() target_image = imageio.imread('celeba-subset/000001.jpg') print('Target image shape: ', target_image.shape) eyepad.fit_image(target_image); Target image shape: (218, 178, 3) ### Transform a test image img = imageio.imread('test-face.png') print('Original Image shape: ', img.shape) img_tr = eyepad.transform(img) print('Transformed Image shape: ', img_tr.shape) Original Image shape: (350, 350, 3) Transformed Image shape: (218, 178, 3) import matplotlib.pyplot as plt fig = plt.figure(figsize=(12, 5)) ax = fig.add_subplot(1, 3, 1) ax.imshow(target_image) ax.set_title('Target', size=20) ax = fig.add_subplot(1, 3, 2) ax.imshow(img) ax.set_title('Original', size=20) ax = fig.add_subplot(1, 3, 3) ax.imshow(img_tr) ax.set_title('Aligned to Target', size=20) plt.show()","title":"Example 1 (fit_image)"},{"location":"user_guide/image/eyepad_align/#example-2-fit_directory","text":"In typical computer vision and deep learning tasks, we are interested in aligning all images in a dataset and across dataset to allow cross-dataset comparisons of face image classifiers. Using the fit_directory method, we can fit EyepadAlign such that the average landmarks are computed from all images in a directory. Then, we can use these average landmarks to transform new images based on the average eye location. import imageio from mlxtend.image import EyepadAlign ### Setting verbosity to 1 will print a progress bar eyepad = EyepadAlign(verbose=1) eyepad.fit_directory(target_img_dir='celeba-subset/', target_width=178, target_height=218, file_extension='.jpg') img = imageio.imread('test-face.png') img_tr = eyepad.transform(img) print('Transformed Image shape: ', img_tr.shape) Pre-Checking directory for consistent image dimensions... 0% [#########] 100% | ETA: 00:00:00 Fitting the average facial landmarks for 9 face images Total time elapsed: 00:00:00 0% [##### ] 100% | ETA: 00:00:00/Users/sebastian/code/mlxtend/mlxtend/image/extract_face_landmarks.py:66: UserWarning: No face detected. warnings.warn('No face detected.') /Users/sebastian/code/mlxtend/mlxtend/image/eyepad_align.py:188: UserWarning: No face detected in image 000004.jpg. Image ignored. % f) 0% [#########] 100% | ETA: 00:00:00 Total time elapsed: 00:00:00 /Users/sebastian/code/mlxtend/mlxtend/image/eyepad_align.py:188: UserWarning: No face detected in image 000003.jpg. Image ignored. % f) Transformed Image shape: (218, 178, 3) Note that a warning is displayed if no face landmarks could be detected. In these cases, the image is skipped and not included in the average landmark computation. E.g., 000003.jpg would be such an example: import matplotlib.pyplot as plt img_no_landmarks = imageio.imread('celeba-subset/000003.jpg') plt.imshow(img_no_landmarks) plt.show() Finally, the an example of aligning the original face image to the average landmarks in the dataset is shown below: fig = plt.figure(figsize=(10, 4)) ax = fig.add_subplot(1, 3, 1) ax.scatter(eyepad.target_landmarks_[:, 0], eyepad.target_landmarks_[:, 1], alpha=0.8) ax.set_title('Target Landmarks', size=20) ax.set_xlim(0, 178) ax.set_ylim(218, 0) ax = fig.add_subplot(1, 3, 2) ax.imshow(img) ax.set_title('Original', size=20) ax = fig.add_subplot(1, 3, 3) ax.imshow(img_tr) ax.set_title('Aligned to Target', size=20) plt.tight_layout() plt.show()","title":"Example 2 (fit_directory)"},{"location":"user_guide/image/eyepad_align/#example-3-fit_values","text":"For more customization, landmarks can also be passed manually. import imageio from mlxtend.image import EyepadAlign import numpy as np celeba_landmarks = np.array([[45, 113], [45, 124], [46, 136], [49, 147], [53, 158], [60, 167], [68, 175], [77, 182], [88, 184], [99, 182], [108, 175], [117, 167], [124, 158], [128, 147], [131, 136], [132, 124], [132, 113], [53, 104], [58, 99], [66, 98], [73, 99], [80, 102], [95, 102], [103, 99], [110, 98], [118, 99], [123, 104], [88, 111], [88, 119], [88, 126], [88, 134], [79, 139], [83, 140], [88, 142], [92, 140], [96, 139], [62, 112], [66, 109], [72, 109], [77, 113], [71, 114], [66, 114], [99, 113], [104, 109], [110, 109], [115, 112], [110, 114], [105, 114], [71, 152], [77, 150], [83, 149], [88, 150], [92, 149], [99, 150], [105, 152], [99, 159], [93, 162], [88, 163], [83, 162], [77, 159], [73, 153], [83, 152], [88, 153], [93, 152], [103, 153], [93, 156], [88, 157], [83, 156]]) eyepad = EyepadAlign(verbose=1) eyepad.fit_values(target_landmarks=celeba_landmarks, target_width=178, target_height=218); import matplotlib.pyplot as plt img = imageio.imread('test-face.png') img_tr = eyepad.transform(img) print('Transformed Image shape: ', img_tr.shape) fig = plt.figure(figsize=(10, 4)) ax = fig.add_subplot(1, 3, 1) ax.scatter(eyepad.target_landmarks_[:, 0], eyepad.target_landmarks_[:, 1], alpha=0.8) ax.set_title('Target Landmarks', size=20) ax.set_xlim(0, 178) ax.set_ylim(218, 0) ax = fig.add_subplot(1, 3, 2) ax.imshow(img) ax.set_title('Original', size=20) ax = fig.add_subplot(1, 3, 3) ax.imshow(img_tr) ax.set_title('Aligned to Target', size=20) plt.tight_layout() plt.show() Transformed Image shape: (218, 178, 3)","title":"Example 3 (fit_values)"},{"location":"user_guide/image/eyepad_align/#example-4-centering-the-nose","text":"In many applications that involve center-cropping, it is helpful to translate the image such that the tip of the nose is located in the center of the image. ### Fit EyepadAlign to dataset to compute average landmarks import imageio from mlxtend.image import EyepadAlign import numpy as np eyepad = EyepadAlign(verbose=1) eyepad.fit_directory(target_img_dir='celeba-subset/', target_width=178, target_height=218, file_extension='.jpg') Pre-Checking directory for consistent image dimensions... 0% [#########] 100% | ETA: 00:00:00 Fitting the average facial landmarks for 9 face images Total time elapsed: 00:00:00 0% [#########] 100% | ETA: 00:00:00 Total time elapsed: 00:00:00 <mlxtend.image.eyepad_align.EyepadAlign at 0x7fa09865d390> ### Use the calculated landmarks to center the nose: center = np.array([178//2, 218//2]) displacement_vector = center - eyepad.target_landmarks_[33] # index 33 is the nose-tip nose_centered_landmarks = eyepad.target_landmarks_ + displacement_vector eyepad_cent_nose = EyepadAlign(verbose=1) eyepad_cent_nose.fit_values(target_landmarks=nose_centered_landmarks, target_width=178, target_height=218); ### Evaluate the results img = imageio.imread('test-face.png') img_tr = eyepad.transform(img) img_nose_centered = eyepad_cent_nose.transform(img) fig = plt.figure(figsize=(10, 4)) ax = fig.add_subplot(1, 3, 1) ax.imshow(img) ax.set_title('Original', size=20) ax = fig.add_subplot(1, 3, 2) ax.imshow(img_tr) ax.set_title('Aligned (Average eye \\n position in dataset)', size=15) ax = fig.add_subplot(1, 3, 3) ax.imshow(img_nose_centered) ax.set_title('Aligned (Nose in image center)', size=15) plt.tight_layout() plt.show()","title":"Example 4 (centering the nose)"},{"location":"user_guide/image/eyepad_align/#api","text":"EyepadAlign(verbose=0) Class to align/transform face images to facial landmarks, based on eye alignment. A scaling factor is computed based on distance between the left and right eye, such that the transformed face image will have the same eye distance as a reference face image. A transformation is performed based on the eyes' center point. to align the face based on the reference eye location. Finally, the transformed image is padded with zeros to match the desired final image size. Parameters verbose : int (default=0) Verbose level to display the progress bar and log messages. Setting verbose=1 will print a progress bar upon calling fit_directory . Attributes target_landmarks_ : target landmarks to transform new face images to. Depending on the chosen fit parameters, it can be either (1) assigned to pre-fit shapes, (2) computed from a single face image (3) computed as the mean of face landmarks from all face images in a file directory of face images. eye_distance_ : the distance between left and right eyes in the target landmarks. target_height_ : the height of the transformed output image. target_width_ : the width of the transformed output image. Returns self : object Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/image/eyepad_align/","title":"API"},{"location":"user_guide/image/eyepad_align/#methods","text":"fit_directory(target_img_dir, target_height, target_width, file_extension='.jpg', pre_check=True) Calculates the average landmarks for all face images in a directory which will then be set as the target landmark set. Arguments target_img_dir : str Directory containing the images target_height : int Expected image height of the images in the directory target_width : int Expected image width of the images in the directory file_extension str (default='.jpg'): File extension of the image files. pre_check Bool (default=True): Checks that each image has the dimensions specificed via target_height and target_width on the whole directory first to identify potential issues that are recommended to be fixed before proceeding. Raises a warning for each image if dimensions differ from the ones specified and expected. Returns self : object fit_image(target_image) Derives facial landmarks from a target image. Arguments target_image : uint8 numpy.array, shape=[width, height, channels] NumPy array representation of the image data. Returns self : object fit_values(target_landmarks, target_width, target_height) Used for determining the eye location from pre-defined landmark arrays, eliminating the need for re-computing the average landmarks on a target image or image directory. Arguments target_landmarks : np.array, shape=(height, width) NumPy array containing the locations of the facial landmarks as determined by mlxtend.image.extract_face_landmarks target_height : int image height target_width : int image width Returns self : object transform(img) transforms a single face image (img) to the target landmarks based on the location of the eyes by scaling, translation and cropping (if needed): (1) Scaling the image so that the distance of the two eyes in the given image (img) matches the distance of the two eyes in the target landmarks. (2) Translation is performed based on the middle point between the two eyes. Arguments img : np.array, shape=(height, width, channels) Input image to be transformed. Returns self : object","title":"Methods"},{"location":"user_guide/math/num_combinations/","text":"num_combinations: combinations for creating subsequences of k elements A function to calculate the number of combinations for creating subsequences of k elements out of a sequence with n elements. from mlxtend.math import num_combinations Overview Combinations are selections of items from a collection regardless of the order in which they appear (in contrast to permutations). For example, let's consider a combination of 3 elements (k=3) from a collection of 5 elements (n=5): collection: {1, 2, 3, 4, 5} combination 1a: {1, 3, 5} combination 1b: {1, 5, 3} combination 1c: {3, 5, 1} ... combination 2: {1, 3, 4} In the example above the combinations 1a, 1b, and 1c, are the \"same combination\" and counted as \"1 possible way to combine items 1, 3, and 5\" -- in combinations, the order does not matter. The number of ways to combine elements ( without replacement ) from a collection with size n into subsets of size k is computed via the binomial coefficient (\" n choose k \"): \\begin{pmatrix} n \\\\ k \\end{pmatrix} = \\frac{n(n-1)\\ldots(n-k+1)}{k(k-1)\\dots1} = \\frac{n!}{k!(n-k)!} To compute the number of combinations with replacement , the following, alternative equation is used (\" n multichoose k \"): \\begin{pmatrix} n \\\\ k \\end{pmatrix} = \\begin{pmatrix} n + k -1 \\\\ k \\end{pmatrix} References https://en.wikipedia.org/wiki/Combination Example 1 - Compute the number of combinations from mlxtend.math import num_combinations c = num_combinations(n=20, k=8, with_replacement=False) print('Number of ways to combine 20 elements' ' into 8 subelements: %d' % c) Number of ways to combine 20 elements into 8 subelements: 125970 from mlxtend.math import num_combinations c = num_combinations(n=20, k=8, with_replacement=True) print('Number of ways to combine 20 elements' ' into 8 subelements (with replacement): %d' % c) Number of ways to combine 20 elements into 8 subelements (with replacement): 2220075 Example 2 - A progress tracking use-case It is often quite useful to track the progress of a computational expensive tasks to estimate its runtime. Here, the num_combination function can be used to compute the maximum number of loops of a combinations iterable from itertools: import itertools import sys import time from mlxtend.math import num_combinations items = {1, 2, 3, 4, 5, 6, 7, 8} max_iter = num_combinations(n=len(items), k=3, with_replacement=False) for idx, i in enumerate(itertools.combinations(items, r=3)): # do some computation with itemset i time.sleep(0.1) sys.stdout.write('\\rProgress: %d/%d' % (idx + 1, max_iter)) sys.stdout.flush() Progress: 56/56 API num_combinations(n, k, with_replacement=False) Function to calculate the number of possible combinations. Parameters n : int Total number of items. k : int Number of elements of the target itemset. with_replacement : bool (default: False) Allows repeated elements if True. Returns comb : int Number of possible combinations. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/math/num_combinations/","title":"num_combinations: combinations for creating subsequences of k elements"},{"location":"user_guide/math/num_combinations/#num_combinations-combinations-for-creating-subsequences-of-k-elements","text":"A function to calculate the number of combinations for creating subsequences of k elements out of a sequence with n elements. from mlxtend.math import num_combinations","title":"num_combinations: combinations for creating subsequences of k elements"},{"location":"user_guide/math/num_combinations/#overview","text":"Combinations are selections of items from a collection regardless of the order in which they appear (in contrast to permutations). For example, let's consider a combination of 3 elements (k=3) from a collection of 5 elements (n=5): collection: {1, 2, 3, 4, 5} combination 1a: {1, 3, 5} combination 1b: {1, 5, 3} combination 1c: {3, 5, 1} ... combination 2: {1, 3, 4} In the example above the combinations 1a, 1b, and 1c, are the \"same combination\" and counted as \"1 possible way to combine items 1, 3, and 5\" -- in combinations, the order does not matter. The number of ways to combine elements ( without replacement ) from a collection with size n into subsets of size k is computed via the binomial coefficient (\" n choose k \"): \\begin{pmatrix} n \\\\ k \\end{pmatrix} = \\frac{n(n-1)\\ldots(n-k+1)}{k(k-1)\\dots1} = \\frac{n!}{k!(n-k)!} To compute the number of combinations with replacement , the following, alternative equation is used (\" n multichoose k \"): \\begin{pmatrix} n \\\\ k \\end{pmatrix} = \\begin{pmatrix} n + k -1 \\\\ k \\end{pmatrix}","title":"Overview"},{"location":"user_guide/math/num_combinations/#references","text":"https://en.wikipedia.org/wiki/Combination","title":"References"},{"location":"user_guide/math/num_combinations/#example-1-compute-the-number-of-combinations","text":"from mlxtend.math import num_combinations c = num_combinations(n=20, k=8, with_replacement=False) print('Number of ways to combine 20 elements' ' into 8 subelements: %d' % c) Number of ways to combine 20 elements into 8 subelements: 125970 from mlxtend.math import num_combinations c = num_combinations(n=20, k=8, with_replacement=True) print('Number of ways to combine 20 elements' ' into 8 subelements (with replacement): %d' % c) Number of ways to combine 20 elements into 8 subelements (with replacement): 2220075","title":"Example 1 - Compute the number of combinations"},{"location":"user_guide/math/num_combinations/#example-2-a-progress-tracking-use-case","text":"It is often quite useful to track the progress of a computational expensive tasks to estimate its runtime. Here, the num_combination function can be used to compute the maximum number of loops of a combinations iterable from itertools: import itertools import sys import time from mlxtend.math import num_combinations items = {1, 2, 3, 4, 5, 6, 7, 8} max_iter = num_combinations(n=len(items), k=3, with_replacement=False) for idx, i in enumerate(itertools.combinations(items, r=3)): # do some computation with itemset i time.sleep(0.1) sys.stdout.write('\\rProgress: %d/%d' % (idx + 1, max_iter)) sys.stdout.flush() Progress: 56/56","title":"Example 2 - A progress tracking use-case"},{"location":"user_guide/math/num_combinations/#api","text":"num_combinations(n, k, with_replacement=False) Function to calculate the number of possible combinations. Parameters n : int Total number of items. k : int Number of elements of the target itemset. with_replacement : bool (default: False) Allows repeated elements if True. Returns comb : int Number of possible combinations. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/math/num_combinations/","title":"API"},{"location":"user_guide/math/num_permutations/","text":"num_permutations: number of permutations for creating subsequences of k elements A function to calculate the number of permutations for creating subsequences of k elements out of a sequence with n elements. from mlxtend.math import num_permutations Overview Permutations are selections of items from a collection with regard to the order in which they appear (in contrast to combinations). For example, let's consider a permutation of 3 elements (k=3) from a collection of 5 elements (n=5): collection: {1, 2, 3, 4, 5} combination 1a: {1, 3, 5} combination 1b: {1, 5, 3} combination 1c: {3, 5, 1} ... combination 2: {1, 3, 4} In the example above the permutations 1a, 1b, and 1c, are the \"same combination\" but distinct permutations -- in combinations, the order does not matter, but in permutation it does matter. The number of ways to combine elements ( without replacement ) from a collection with size n into subsets of size k is computed via the binomial coefficient (\" n choose k \"): k!\\begin{pmatrix} n \\\\ k \\end{pmatrix} = k! \\cdot \\frac{n!}{k!(n-k)!} = \\frac{n!}{(n-k)!} To compute the number of permutations with replacement , we simply need to compute n^k . References https://en.wikipedia.org/wiki/Permutation Example 1 - Compute the number of permutations from mlxtend.math import num_permutations c = num_permutations(n=20, k=8, with_replacement=False) print('Number of ways to permute 20 elements' ' into 8 subelements: %d' % c) Number of ways to permute 20 elements into 8 subelements: 5079110400 from mlxtend.math import num_permutations c = num_permutations(n=20, k=8, with_replacement=True) print('Number of ways to combine 20 elements' ' into 8 subelements (with replacement): %d' % c) Number of ways to combine 20 elements into 8 subelements (with replacement): 25600000000 Example 2 - A progress tracking use-case It is often quite useful to track the progress of a computational expensive tasks to estimate its runtime. Here, the num_combination function can be used to compute the maximum number of loops of a permutations iterable from itertools: import itertools import sys import time from mlxtend.math import num_permutations items = {1, 2, 3, 4, 5, 6, 7, 8} max_iter = num_permutations(n=len(items), k=3, with_replacement=False) for idx, i in enumerate(itertools.permutations(items, r=3)): # do some computation with itemset i time.sleep(0.01) sys.stdout.write('\\rProgress: %d/%d' % (idx + 1, max_iter)) sys.stdout.flush() Progress: 336/336 API num_permutations(n, k, with_replacement=False) Function to calculate the number of possible permutations. Parameters n : int Total number of items. k : int Number of elements of the target itemset. with_replacement : bool Allows repeated elements if True. Returns permut : int Number of possible permutations. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/math/num_permutations/","title":"num_permutations: number of permutations for creating subsequences of k elements"},{"location":"user_guide/math/num_permutations/#num_permutations-number-of-permutations-for-creating-subsequences-of-k-elements","text":"A function to calculate the number of permutations for creating subsequences of k elements out of a sequence with n elements. from mlxtend.math import num_permutations","title":"num_permutations: number of permutations for creating subsequences of k elements"},{"location":"user_guide/math/num_permutations/#overview","text":"Permutations are selections of items from a collection with regard to the order in which they appear (in contrast to combinations). For example, let's consider a permutation of 3 elements (k=3) from a collection of 5 elements (n=5): collection: {1, 2, 3, 4, 5} combination 1a: {1, 3, 5} combination 1b: {1, 5, 3} combination 1c: {3, 5, 1} ... combination 2: {1, 3, 4} In the example above the permutations 1a, 1b, and 1c, are the \"same combination\" but distinct permutations -- in combinations, the order does not matter, but in permutation it does matter. The number of ways to combine elements ( without replacement ) from a collection with size n into subsets of size k is computed via the binomial coefficient (\" n choose k \"): k!\\begin{pmatrix} n \\\\ k \\end{pmatrix} = k! \\cdot \\frac{n!}{k!(n-k)!} = \\frac{n!}{(n-k)!} To compute the number of permutations with replacement , we simply need to compute n^k .","title":"Overview"},{"location":"user_guide/math/num_permutations/#references","text":"https://en.wikipedia.org/wiki/Permutation","title":"References"},{"location":"user_guide/math/num_permutations/#example-1-compute-the-number-of-permutations","text":"from mlxtend.math import num_permutations c = num_permutations(n=20, k=8, with_replacement=False) print('Number of ways to permute 20 elements' ' into 8 subelements: %d' % c) Number of ways to permute 20 elements into 8 subelements: 5079110400 from mlxtend.math import num_permutations c = num_permutations(n=20, k=8, with_replacement=True) print('Number of ways to combine 20 elements' ' into 8 subelements (with replacement): %d' % c) Number of ways to combine 20 elements into 8 subelements (with replacement): 25600000000","title":"Example 1 - Compute the number of permutations"},{"location":"user_guide/math/num_permutations/#example-2-a-progress-tracking-use-case","text":"It is often quite useful to track the progress of a computational expensive tasks to estimate its runtime. Here, the num_combination function can be used to compute the maximum number of loops of a permutations iterable from itertools: import itertools import sys import time from mlxtend.math import num_permutations items = {1, 2, 3, 4, 5, 6, 7, 8} max_iter = num_permutations(n=len(items), k=3, with_replacement=False) for idx, i in enumerate(itertools.permutations(items, r=3)): # do some computation with itemset i time.sleep(0.01) sys.stdout.write('\\rProgress: %d/%d' % (idx + 1, max_iter)) sys.stdout.flush() Progress: 336/336","title":"Example 2 - A progress tracking use-case"},{"location":"user_guide/math/num_permutations/#api","text":"num_permutations(n, k, with_replacement=False) Function to calculate the number of possible permutations. Parameters n : int Total number of items. k : int Number of elements of the target itemset. with_replacement : bool Allows repeated elements if True. Returns permut : int Number of possible permutations. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/math/num_permutations/","title":"API"},{"location":"user_guide/math/vectorspace_dimensionality/","text":"vectorspace_dimensionality: compute the number of dimensions that a set of vectors spans A function to compute the number of dimensions a set of vectors (arranged as columns in a matrix) spans. from mlxtend.math import vectorspace_dimensionality Overview Given a set of vectors, arranged as columns in a matrix, the vectorspace_dimensionality computes the number of dimensions (i.e., hyper-volume) that the vectorspace spans using the Gram-Schmidt process [1]. In particular, since the Gram-Schmidt process yields vectors that are zero or normalized to 1 (i.e., an orthonormal vectorset if the input was a set of linearly independent vectors), the sum of the vector norms corresponds to the number of dimensions of a vectorset. References [1] https://en.wikipedia.org/wiki/Gram\u2013Schmidt_process Example 1 - Compute the dimensions of a vectorspace Let's assume we have the two basis vectors x=[1 \\;\\;\\; 0]^T and y=[0\\;\\;\\; 1]^T as columns in a matrix. Due to the linear independence of the two vectors, the space that they span is naturally a plane (2D space): import numpy as np from mlxtend.math import vectorspace_dimensionality a = np.array([[1, 0], [0, 1]]) vectorspace_dimensionality(a) 2 However, if one vector is a linear combination of the other, it's intuitive to see that the space the vectorset describes is merely a line, aka a 1D space: b = np.array([[1, 2], [0, 0]]) vectorspace_dimensionality(a) 2 If 3 vectors are all linearly independent of each other, the dimensionality of the vector space is a volume (i.e., a 3D space): d = np.array([[1, 9, 1], [3, 2, 2], [5, 4, 3]]) vectorspace_dimensionality(d) 3 Again, if a pair of vectors is linearly dependent (here: the 1st and the 2nd row), this reduces the dimensionality by 1: c = np.array([[1, 2, 1], [3, 6, 2], [5, 10, 3]]) vectorspace_dimensionality(c) 2 API vectorspace_dimensionality(ary) Computes the hyper-volume spanned by a vector set Parameters ary : array-like, shape=[num_vectors, num_vectors] An orthogonal set of vectors (arranged as columns in a matrix) Returns dimensions : int An integer indicating the \"dimensionality\" hyper-volume spanned by the vector set Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/math/vectorspace_dimensionality/","title":"vectorspace_dimensionality: compute the number of dimensions that a set of vectors spans"},{"location":"user_guide/math/vectorspace_dimensionality/#vectorspace_dimensionality-compute-the-number-of-dimensions-that-a-set-of-vectors-spans","text":"A function to compute the number of dimensions a set of vectors (arranged as columns in a matrix) spans. from mlxtend.math import vectorspace_dimensionality","title":"vectorspace_dimensionality: compute the number of dimensions that a set of vectors spans"},{"location":"user_guide/math/vectorspace_dimensionality/#overview","text":"Given a set of vectors, arranged as columns in a matrix, the vectorspace_dimensionality computes the number of dimensions (i.e., hyper-volume) that the vectorspace spans using the Gram-Schmidt process [1]. In particular, since the Gram-Schmidt process yields vectors that are zero or normalized to 1 (i.e., an orthonormal vectorset if the input was a set of linearly independent vectors), the sum of the vector norms corresponds to the number of dimensions of a vectorset.","title":"Overview"},{"location":"user_guide/math/vectorspace_dimensionality/#references","text":"[1] https://en.wikipedia.org/wiki/Gram\u2013Schmidt_process","title":"References"},{"location":"user_guide/math/vectorspace_dimensionality/#example-1-compute-the-dimensions-of-a-vectorspace","text":"Let's assume we have the two basis vectors x=[1 \\;\\;\\; 0]^T and y=[0\\;\\;\\; 1]^T as columns in a matrix. Due to the linear independence of the two vectors, the space that they span is naturally a plane (2D space): import numpy as np from mlxtend.math import vectorspace_dimensionality a = np.array([[1, 0], [0, 1]]) vectorspace_dimensionality(a) 2 However, if one vector is a linear combination of the other, it's intuitive to see that the space the vectorset describes is merely a line, aka a 1D space: b = np.array([[1, 2], [0, 0]]) vectorspace_dimensionality(a) 2 If 3 vectors are all linearly independent of each other, the dimensionality of the vector space is a volume (i.e., a 3D space): d = np.array([[1, 9, 1], [3, 2, 2], [5, 4, 3]]) vectorspace_dimensionality(d) 3 Again, if a pair of vectors is linearly dependent (here: the 1st and the 2nd row), this reduces the dimensionality by 1: c = np.array([[1, 2, 1], [3, 6, 2], [5, 10, 3]]) vectorspace_dimensionality(c) 2","title":"Example 1 - Compute the dimensions of a vectorspace"},{"location":"user_guide/math/vectorspace_dimensionality/#api","text":"vectorspace_dimensionality(ary) Computes the hyper-volume spanned by a vector set Parameters ary : array-like, shape=[num_vectors, num_vectors] An orthogonal set of vectors (arranged as columns in a matrix) Returns dimensions : int An integer indicating the \"dimensionality\" hyper-volume spanned by the vector set Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/math/vectorspace_dimensionality/","title":"API"},{"location":"user_guide/math/vectorspace_orthonormalization/","text":"vectorspace_orthonormalization: Converts a set of linearly independent vectors to a set of orthonormal basis vectors A function that converts a set of linearly independent vectors to a set of orthonormal basis vectors. from mlxtend.math import vectorspace_orthonormalization Overview The vectorspace_orthonormalization converts a set linearly independent vectors to a set of orthonormal basis vectors using the Gram-Schmidt process [1]. References [1] https://en.wikipedia.org/wiki/Gram\u2013Schmidt_process Example 1 - Convert a set of vector to an orthonormal basis Note that to convert a set of linearly independent vectors into a set of orthonormal basis vectors, the vectorspace_orthonormalization function expects the vectors to be arranged as columns of a matrix (here: NumPy array). Please keep in mind that the vectorspace_orthonormalization function also works for non-linearly independent vector sets; however, the resulting vectorset won't be orthonormal as a result. An easy way to check whether all vectors in the input set are linearly independent is to use the numpy.linalg.det (determinant) function. import numpy as np from mlxtend.math import vectorspace_orthonormalization a = np.array([[2, 0, 4, 12], [0, 2, 16, 4], [4, 16, 6, 2], [2, -12, 4, 6]]) s = '' if np.linalg.det(a) == 0.0: s = ' not' print('Input vectors are%s linearly independent' % s) vectorspace_orthonormalization(a) Input vectors are linearly independent array([[ 0.40824829, -0.1814885 , 0.04982278, 0.89325973], [ 0. , 0.1088931 , 0.99349591, -0.03328918], [ 0.81649658, 0.50816781, -0.06462163, -0.26631346], [ 0.40824829, -0.83484711, 0.07942048, -0.36063281]]) Note that scaling the inputs equally by a factor should leave the results unchanged: vectorspace_orthonormalization(a/2) array([[ 0.40824829, -0.1814885 , 0.04982278, 0.89325973], [ 0. , 0.1088931 , 0.99349591, -0.03328918], [ 0.81649658, 0.50816781, -0.06462163, -0.26631346], [ 0.40824829, -0.83484711, 0.07942048, -0.36063281]]) However, in case of linear dependence (the second column is a linear combination of the first column in the example below), the vector elements of one of the dependent vectors will become zero. (For a pair of linear dependent vectors, the one with the larger column index will be the one that's zero-ed.) a[:, 1] = a[:, 0] * 2 vectorspace_orthonormalization(a) array([[ 0.40824829, 0. , 0.04155858, 0.82364839], [ 0. , 0. , 0.99740596, -0.06501108], [ 0.81649658, 0. , -0.04155858, -0.52008861], [ 0.40824829, 0. , 0.04155858, 0.21652883]]) API vectorspace_orthonormalization(ary, eps=1e-13) Transforms a set of column vectors to a orthonormal basis. Given a set of orthogonal vectors, this functions converts such column vectors, arranged in a matrix, into orthonormal basis vectors. Parameters ary : array-like, shape=[num_vectors, num_vectors] An orthogonal set of vectors (arranged as columns in a matrix) eps : float (default: 1e-13) A small tolerance value to determine whether the vector norm is zero or not. Returns arr : array-like, shape=[num_vectors, num_vectors] An orthonormal set of vectors (arranged as columns) Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/math/vectorspace_orthonormalization/","title":"vectorspace_orthonormalization: Converts a set of linearly independent vectors to a set of orthonormal basis vectors"},{"location":"user_guide/math/vectorspace_orthonormalization/#vectorspace_orthonormalization-converts-a-set-of-linearly-independent-vectors-to-a-set-of-orthonormal-basis-vectors","text":"A function that converts a set of linearly independent vectors to a set of orthonormal basis vectors. from mlxtend.math import vectorspace_orthonormalization","title":"vectorspace_orthonormalization: Converts a set of linearly independent vectors to a set of orthonormal basis vectors"},{"location":"user_guide/math/vectorspace_orthonormalization/#overview","text":"The vectorspace_orthonormalization converts a set linearly independent vectors to a set of orthonormal basis vectors using the Gram-Schmidt process [1].","title":"Overview"},{"location":"user_guide/math/vectorspace_orthonormalization/#references","text":"[1] https://en.wikipedia.org/wiki/Gram\u2013Schmidt_process","title":"References"},{"location":"user_guide/math/vectorspace_orthonormalization/#example-1-convert-a-set-of-vector-to-an-orthonormal-basis","text":"Note that to convert a set of linearly independent vectors into a set of orthonormal basis vectors, the vectorspace_orthonormalization function expects the vectors to be arranged as columns of a matrix (here: NumPy array). Please keep in mind that the vectorspace_orthonormalization function also works for non-linearly independent vector sets; however, the resulting vectorset won't be orthonormal as a result. An easy way to check whether all vectors in the input set are linearly independent is to use the numpy.linalg.det (determinant) function. import numpy as np from mlxtend.math import vectorspace_orthonormalization a = np.array([[2, 0, 4, 12], [0, 2, 16, 4], [4, 16, 6, 2], [2, -12, 4, 6]]) s = '' if np.linalg.det(a) == 0.0: s = ' not' print('Input vectors are%s linearly independent' % s) vectorspace_orthonormalization(a) Input vectors are linearly independent array([[ 0.40824829, -0.1814885 , 0.04982278, 0.89325973], [ 0. , 0.1088931 , 0.99349591, -0.03328918], [ 0.81649658, 0.50816781, -0.06462163, -0.26631346], [ 0.40824829, -0.83484711, 0.07942048, -0.36063281]]) Note that scaling the inputs equally by a factor should leave the results unchanged: vectorspace_orthonormalization(a/2) array([[ 0.40824829, -0.1814885 , 0.04982278, 0.89325973], [ 0. , 0.1088931 , 0.99349591, -0.03328918], [ 0.81649658, 0.50816781, -0.06462163, -0.26631346], [ 0.40824829, -0.83484711, 0.07942048, -0.36063281]]) However, in case of linear dependence (the second column is a linear combination of the first column in the example below), the vector elements of one of the dependent vectors will become zero. (For a pair of linear dependent vectors, the one with the larger column index will be the one that's zero-ed.) a[:, 1] = a[:, 0] * 2 vectorspace_orthonormalization(a) array([[ 0.40824829, 0. , 0.04155858, 0.82364839], [ 0. , 0. , 0.99740596, -0.06501108], [ 0.81649658, 0. , -0.04155858, -0.52008861], [ 0.40824829, 0. , 0.04155858, 0.21652883]])","title":"Example 1 - Convert a set of vector to an orthonormal basis"},{"location":"user_guide/math/vectorspace_orthonormalization/#api","text":"vectorspace_orthonormalization(ary, eps=1e-13) Transforms a set of column vectors to a orthonormal basis. Given a set of orthogonal vectors, this functions converts such column vectors, arranged in a matrix, into orthonormal basis vectors. Parameters ary : array-like, shape=[num_vectors, num_vectors] An orthogonal set of vectors (arranged as columns in a matrix) eps : float (default: 1e-13) A small tolerance value to determine whether the vector norm is zero or not. Returns arr : array-like, shape=[num_vectors, num_vectors] An orthonormal set of vectors (arranged as columns) Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/math/vectorspace_orthonormalization/","title":"API"},{"location":"user_guide/plotting/category_scatter/","text":"Scategory_scatter: Create a scatterplot with categories in different colors A function to quickly produce a scatter plot colored by categories from a pandas DataFrame or NumPy ndarray object. from mlxtend.general import category_scatter Overview References - Example 1 - Category Scatter from Pandas DataFrames import pandas as pd from io import StringIO csvfile = \"\"\"label,x,y class1,10.0,8.04 class1,10.5,7.30 class2,8.3,5.5 class2,8.1,5.9 class3,3.5,3.5 class3,3.8,5.1\"\"\" df = pd.read_csv(StringIO(csvfile)) df label x y 0 class1 10.0 8.04 1 class1 10.5 7.30 2 class2 8.3 5.50 3 class2 8.1 5.90 4 class3 3.5 3.50 5 class3 3.8 5.10 Plotting the data where the categories are determined by the unique values in the label column label_col . The x and y values are simply the column names of the DataFrame that we want to plot. import matplotlib.pyplot as plt from mlxtend.plotting import category_scatter fig = category_scatter(x='x', y='y', label_col='label', data=df, legend_loc='upper left') Example 2 - Category Scatter from NumPy Arrays import numpy as np from io import BytesIO csvfile = \"\"\"1,10.0,8.04 1,10.5,7.30 2,8.3,5.5 2,8.1,5.9 3,3.5,3.5 3,3.8,5.1\"\"\" ary = np.genfromtxt(BytesIO(csvfile.encode()), delimiter=',') ary array([[ 1. , 10. , 8.04], [ 1. , 10.5 , 7.3 ], [ 2. , 8.3 , 5.5 ], [ 2. , 8.1 , 5.9 ], [ 3. , 3.5 , 3.5 ], [ 3. , 3.8 , 5.1 ]]) Now, pretending that the first column represents the labels, and the second and third column represent the x and y values, respectively. import matplotlib.pyplot as plt from mlxtend.plotting import category_scatter fix = category_scatter(x=1, y=2, label_col=0, data=ary, legend_loc='upper left') API category_scatter(x, y, label_col, data, markers='sxo^v', colors=('blue', 'green', 'red', 'purple', 'gray', 'cyan'), alpha=0.7, markersize=20.0, legend_loc='best') Scatter plot to plot categories in different colors/markerstyles. Parameters x : str or int DataFrame column name of the x-axis values or integer for the numpy ndarray column index. y : str DataFrame column name of the y-axis values or integer for the numpy ndarray column index data : Pandas DataFrame object or NumPy ndarray. markers : str Markers that are cycled through the label category. colors : tuple Colors that are cycled through the label category. alpha : float (default: 0.7) Parameter to control the transparency. markersize : float (default` : 20.0) Parameter to control the marker size. legend_loc : str (default: 'best') Location of the plot legend {best, upper left, upper right, lower left, lower right} No legend if legend_loc=False Returns fig : matplotlig.pyplot figure object Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/plotting/category_scatter/","title":"Scategory_scatter: Create a scatterplot with categories in different colors"},{"location":"user_guide/plotting/category_scatter/#scategory_scatter-create-a-scatterplot-with-categories-in-different-colors","text":"A function to quickly produce a scatter plot colored by categories from a pandas DataFrame or NumPy ndarray object. from mlxtend.general import category_scatter","title":"Scategory_scatter: Create a scatterplot with categories in different colors"},{"location":"user_guide/plotting/category_scatter/#overview","text":"","title":"Overview"},{"location":"user_guide/plotting/category_scatter/#references","text":"-","title":"References"},{"location":"user_guide/plotting/category_scatter/#example-1-category-scatter-from-pandas-dataframes","text":"import pandas as pd from io import StringIO csvfile = \"\"\"label,x,y class1,10.0,8.04 class1,10.5,7.30 class2,8.3,5.5 class2,8.1,5.9 class3,3.5,3.5 class3,3.8,5.1\"\"\" df = pd.read_csv(StringIO(csvfile)) df label x y 0 class1 10.0 8.04 1 class1 10.5 7.30 2 class2 8.3 5.50 3 class2 8.1 5.90 4 class3 3.5 3.50 5 class3 3.8 5.10 Plotting the data where the categories are determined by the unique values in the label column label_col . The x and y values are simply the column names of the DataFrame that we want to plot. import matplotlib.pyplot as plt from mlxtend.plotting import category_scatter fig = category_scatter(x='x', y='y', label_col='label', data=df, legend_loc='upper left')","title":"Example 1 - Category Scatter from Pandas DataFrames"},{"location":"user_guide/plotting/category_scatter/#example-2-category-scatter-from-numpy-arrays","text":"import numpy as np from io import BytesIO csvfile = \"\"\"1,10.0,8.04 1,10.5,7.30 2,8.3,5.5 2,8.1,5.9 3,3.5,3.5 3,3.8,5.1\"\"\" ary = np.genfromtxt(BytesIO(csvfile.encode()), delimiter=',') ary array([[ 1. , 10. , 8.04], [ 1. , 10.5 , 7.3 ], [ 2. , 8.3 , 5.5 ], [ 2. , 8.1 , 5.9 ], [ 3. , 3.5 , 3.5 ], [ 3. , 3.8 , 5.1 ]]) Now, pretending that the first column represents the labels, and the second and third column represent the x and y values, respectively. import matplotlib.pyplot as plt from mlxtend.plotting import category_scatter fix = category_scatter(x=1, y=2, label_col=0, data=ary, legend_loc='upper left')","title":"Example 2 - Category Scatter from NumPy Arrays"},{"location":"user_guide/plotting/category_scatter/#api","text":"category_scatter(x, y, label_col, data, markers='sxo^v', colors=('blue', 'green', 'red', 'purple', 'gray', 'cyan'), alpha=0.7, markersize=20.0, legend_loc='best') Scatter plot to plot categories in different colors/markerstyles. Parameters x : str or int DataFrame column name of the x-axis values or integer for the numpy ndarray column index. y : str DataFrame column name of the y-axis values or integer for the numpy ndarray column index data : Pandas DataFrame object or NumPy ndarray. markers : str Markers that are cycled through the label category. colors : tuple Colors that are cycled through the label category. alpha : float (default: 0.7) Parameter to control the transparency. markersize : float (default` : 20.0) Parameter to control the marker size. legend_loc : str (default: 'best') Location of the plot legend {best, upper left, upper right, lower left, lower right} No legend if legend_loc=False Returns fig : matplotlig.pyplot figure object Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/plotting/category_scatter/","title":"API"},{"location":"user_guide/plotting/checkerboard_plot/","text":"checkerboard_plot: Create a checkerboard plot in matplotlib Function to plot a checkerboard plot / heat map via matplotlib from mlxtend.plotting import checkerboard_plot Overview Function to plot a checkerboard plot / heat map via matplotlib. References - Example 1 - Default from mlxtend.plotting import checkerboard_plot import matplotlib.pyplot as plt import numpy as np ary = np.random.random((5, 4)) brd = checkerboard_plot(ary) plt.show() Example 2 - Changing colors and labels from mlxtend.plotting import checkerboard_plot import matplotlib.pyplot as plt import numpy as np checkerboard_plot(ary, col_labels=['abc', 'def', 'ghi', 'jkl'], row_labels=['sample %d' % i for i in range(1, 6)], cell_colors=['skyblue', 'whitesmoke'], font_colors=['black', 'black'], figsize=(4.5, 5)) plt.show() API checkerboard_plot(ary, cell_colors=('white', 'black'), font_colors=('black', 'white'), fmt='%.1f', figsize=None, row_labels=None, col_labels=None, fontsize=None) Plot a checkerboard table / heatmap via matplotlib. Parameters ary : array-like, shape = [n, m] A 2D Nnumpy array. cell_colors : tuple or list (default: ('white', 'black')) Tuple or list containing the two colors of the checkerboard pattern. font_colors : tuple or list (default: ('black', 'white')) Font colors corresponding to the cell colors. figsize : tuple (default: (2.5, 2.5)) Height and width of the figure fmt : str (default: '%.1f') Python string formatter for cell values. The default '%.1f' results in floats with 1 digit after the decimal point. Use '%d' to show numbers as integers. row_labels : list (default: None) List of the row labels. Uses the array row indices 0 to n by default. col_labels : list (default: None) List of the column labels. Uses the array column indices 0 to m by default. fontsize : int (default: None) Specifies the font size of the checkerboard table. Uses matplotlib's default if None. Returns fig : matplotlib Figure object. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/plotting/checkerboard_plot/","title":"checkerboard_plot: Create a checkerboard plot in matplotlib"},{"location":"user_guide/plotting/checkerboard_plot/#checkerboard_plot-create-a-checkerboard-plot-in-matplotlib","text":"Function to plot a checkerboard plot / heat map via matplotlib from mlxtend.plotting import checkerboard_plot","title":"checkerboard_plot: Create a checkerboard plot in matplotlib"},{"location":"user_guide/plotting/checkerboard_plot/#overview","text":"Function to plot a checkerboard plot / heat map via matplotlib.","title":"Overview"},{"location":"user_guide/plotting/checkerboard_plot/#references","text":"-","title":"References"},{"location":"user_guide/plotting/checkerboard_plot/#example-1-default","text":"from mlxtend.plotting import checkerboard_plot import matplotlib.pyplot as plt import numpy as np ary = np.random.random((5, 4)) brd = checkerboard_plot(ary) plt.show()","title":"Example 1 - Default"},{"location":"user_guide/plotting/checkerboard_plot/#example-2-changing-colors-and-labels","text":"from mlxtend.plotting import checkerboard_plot import matplotlib.pyplot as plt import numpy as np checkerboard_plot(ary, col_labels=['abc', 'def', 'ghi', 'jkl'], row_labels=['sample %d' % i for i in range(1, 6)], cell_colors=['skyblue', 'whitesmoke'], font_colors=['black', 'black'], figsize=(4.5, 5)) plt.show()","title":"Example 2 - Changing colors and labels"},{"location":"user_guide/plotting/checkerboard_plot/#api","text":"checkerboard_plot(ary, cell_colors=('white', 'black'), font_colors=('black', 'white'), fmt='%.1f', figsize=None, row_labels=None, col_labels=None, fontsize=None) Plot a checkerboard table / heatmap via matplotlib. Parameters ary : array-like, shape = [n, m] A 2D Nnumpy array. cell_colors : tuple or list (default: ('white', 'black')) Tuple or list containing the two colors of the checkerboard pattern. font_colors : tuple or list (default: ('black', 'white')) Font colors corresponding to the cell colors. figsize : tuple (default: (2.5, 2.5)) Height and width of the figure fmt : str (default: '%.1f') Python string formatter for cell values. The default '%.1f' results in floats with 1 digit after the decimal point. Use '%d' to show numbers as integers. row_labels : list (default: None) List of the row labels. Uses the array row indices 0 to n by default. col_labels : list (default: None) List of the column labels. Uses the array column indices 0 to m by default. fontsize : int (default: None) Specifies the font size of the checkerboard table. Uses matplotlib's default if None. Returns fig : matplotlib Figure object. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/plotting/checkerboard_plot/","title":"API"},{"location":"user_guide/plotting/ecdf/","text":"ecdf: Create an empirical cumulative distribution function plot A function to conveniently plot an empirical cumulative distribution function. from mlxtend.ecdf import ecdf Overview A function to conveniently plot an empirical cumulative distribution function (ECDF) and adding percentile thresholds for exploratory data analysis. References - Example 1 - ECDF from mlxtend.data import iris_data from mlxtend.plotting import ecdf import matplotlib.pyplot as plt X, y = iris_data() ax, _, _ = ecdf(x=X[:, 0], x_label='sepal length (cm)') plt.show() Example 2 - Multiple ECDFs from mlxtend.data import iris_data from mlxtend.plotting import ecdf import matplotlib.pyplot as plt X, y = iris_data() # first ecdf x1 = X[:, 0] ax, _, _ = ecdf(x1, x_label='cm') # second ecdf x2 = X[:, 1] ax, _, _ = ecdf(x2, ax=ax) plt.legend(['sepal length', 'sepal width']) plt.show() Example 3 - ECDF with Percentile Thresholds from mlxtend.data import iris_data from mlxtend.plotting import ecdf import matplotlib.pyplot as plt X, y = iris_data() ax, threshold, count = ecdf(x=X[:, 0], x_label='sepal length (cm)', percentile=0.8) plt.show() print('Feature threshold at the 80th percentile:', threshold) print('Number of samples below the threshold:', count) Feature threshold at the 80th percentile: 6.5 Number of samples below the threshold: 120 API ecdf(x, y_label='ECDF', x_label=None, ax=None, percentile=None, ecdf_color=None, ecdf_marker='o', percentile_color='black', percentile_linestyle='--') Plots an Empirical Cumulative Distribution Function Parameters x : array or list, shape=[n_samples,] Array-like object containing the feature values y_label : str (default='ECDF') Text label for the y-axis x_label : str (default=None) Text label for the x-axis ax : matplotlib.axes.Axes (default: None) An existing matplotlib Axes. Creates one if ax=None percentile : float (default=None) Float between 0 and 1 for plotting a percentile threshold line ecdf_color : matplotlib color (default=None) Color for the ECDF plot; uses matplotlib defaults if None ecdf_marker : matplotlib marker (default='o') Marker style for the ECDF plot percentile_color : matplotlib color (default='black') Color for the percentile threshold if percentile is not None percentile_linestyle : matplotlib linestyle (default='--') Line style for the percentile threshold if percentile is not None Returns ax : matplotlib.axes.Axes object percentile_threshold : float Feature threshold at the percentile or None if percentile=None percentile_count : Number of if percentile is not None Number of samples that have a feature less or equal than the feature threshold at a percentile threshold or None if percentile=None Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/plotting/ecdf/","title":"ecdf: Create an empirical cumulative distribution function plot"},{"location":"user_guide/plotting/ecdf/#ecdf-create-an-empirical-cumulative-distribution-function-plot","text":"A function to conveniently plot an empirical cumulative distribution function. from mlxtend.ecdf import ecdf","title":"ecdf: Create an empirical cumulative distribution function plot"},{"location":"user_guide/plotting/ecdf/#overview","text":"A function to conveniently plot an empirical cumulative distribution function (ECDF) and adding percentile thresholds for exploratory data analysis.","title":"Overview"},{"location":"user_guide/plotting/ecdf/#references","text":"-","title":"References"},{"location":"user_guide/plotting/ecdf/#example-1-ecdf","text":"from mlxtend.data import iris_data from mlxtend.plotting import ecdf import matplotlib.pyplot as plt X, y = iris_data() ax, _, _ = ecdf(x=X[:, 0], x_label='sepal length (cm)') plt.show()","title":"Example 1 - ECDF"},{"location":"user_guide/plotting/ecdf/#example-2-multiple-ecdfs","text":"from mlxtend.data import iris_data from mlxtend.plotting import ecdf import matplotlib.pyplot as plt X, y = iris_data() # first ecdf x1 = X[:, 0] ax, _, _ = ecdf(x1, x_label='cm') # second ecdf x2 = X[:, 1] ax, _, _ = ecdf(x2, ax=ax) plt.legend(['sepal length', 'sepal width']) plt.show()","title":"Example 2 - Multiple ECDFs"},{"location":"user_guide/plotting/ecdf/#example-3-ecdf-with-percentile-thresholds","text":"from mlxtend.data import iris_data from mlxtend.plotting import ecdf import matplotlib.pyplot as plt X, y = iris_data() ax, threshold, count = ecdf(x=X[:, 0], x_label='sepal length (cm)', percentile=0.8) plt.show() print('Feature threshold at the 80th percentile:', threshold) print('Number of samples below the threshold:', count) Feature threshold at the 80th percentile: 6.5 Number of samples below the threshold: 120","title":"Example 3 - ECDF with Percentile Thresholds"},{"location":"user_guide/plotting/ecdf/#api","text":"ecdf(x, y_label='ECDF', x_label=None, ax=None, percentile=None, ecdf_color=None, ecdf_marker='o', percentile_color='black', percentile_linestyle='--') Plots an Empirical Cumulative Distribution Function Parameters x : array or list, shape=[n_samples,] Array-like object containing the feature values y_label : str (default='ECDF') Text label for the y-axis x_label : str (default=None) Text label for the x-axis ax : matplotlib.axes.Axes (default: None) An existing matplotlib Axes. Creates one if ax=None percentile : float (default=None) Float between 0 and 1 for plotting a percentile threshold line ecdf_color : matplotlib color (default=None) Color for the ECDF plot; uses matplotlib defaults if None ecdf_marker : matplotlib marker (default='o') Marker style for the ECDF plot percentile_color : matplotlib color (default='black') Color for the percentile threshold if percentile is not None percentile_linestyle : matplotlib linestyle (default='--') Line style for the percentile threshold if percentile is not None Returns ax : matplotlib.axes.Axes object percentile_threshold : float Feature threshold at the percentile or None if percentile=None percentile_count : Number of if percentile is not None Number of samples that have a feature less or equal than the feature threshold at a percentile threshold or None if percentile=None Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/plotting/ecdf/","title":"API"},{"location":"user_guide/plotting/enrichment_plot/","text":"enrichment_plot: create an enrichment plot for cumulative counts A function to plot step plots of cumulative counts. from mlxtend.general import enrichment_plot Overview In enrichment plots, the y-axis can be interpreted as \"how many samples are less or equal to the corresponding x-axis label.\" References - Example 1 - Enrichment Plots from Pandas DataFrames import pandas as pd s1 = [1.1, 1.5] s2 = [2.1, 1.8] s3 = [3.1, 2.1] s4 = [3.9, 2.5] data = [s1, s2, s3, s4] df = pd.DataFrame(data, columns=['X1', 'X2']) df X1 X2 0 1.1 1.5 1 2.1 1.8 2 3.1 2.1 3 3.9 2.5 Plotting the data where the categories are determined by the unique values in the label column label_col . The x and y values are simply the column names of the DataFrame that we want to plot. import matplotlib.pyplot as plt from mlxtend.plotting import enrichment_plot ax = enrichment_plot(df, legend_loc='upper left') API enrichment_plot(df, colors='bgrkcy', markers=' ', linestyles='-', alpha=0.5, lw=2, where='post', grid=True, count_label='Count', xlim='auto', ylim='auto', invert_axes=False, legend_loc='best', ax=None) Plot stacked barplots Parameters df : pandas.DataFrame A pandas DataFrame where columns represent the different categories. colors: str (default: 'bgrcky') The colors of the bars. markers : str (default: ' ') Matplotlib markerstyles, e.g, 'sov' for square,circle, and triangle markers. linestyles : str (default: '-') Matplotlib linestyles, e.g., '-,--' to cycle normal and dashed lines. Note that the different linestyles need to be separated by commas. alpha : float (default: 0.5) Transparency level from 0.0 to 1.0. lw : int or float (default: 2) Linewidth parameter. where : {'post', 'pre', 'mid'} (default: 'post') Starting location of the steps. grid : bool (default: True ) Plots a grid if True. count_label : str (default: 'Count') Label for the \"Count\"-axis. xlim : 'auto' or array-like [min, max] (default: 'auto') Min and maximum position of the x-axis range. ylim : 'auto' or array-like [min, max] (default: 'auto') Min and maximum position of the y-axis range. invert_axes : bool (default: False) Plots count on the x-axis if True. legend_loc : str (default: 'best') Location of the plot legend {best, upper left, upper right, lower left, lower right} No legend if legend_loc=False ax : matplotlib axis, optional (default: None) Use this axis for plotting or make a new one otherwise Returns ax : matplotlib axis Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/plotting/enrichment_plot/","title":"enrichment_plot: create an enrichment plot for cumulative counts"},{"location":"user_guide/plotting/enrichment_plot/#enrichment_plot-create-an-enrichment-plot-for-cumulative-counts","text":"A function to plot step plots of cumulative counts. from mlxtend.general import enrichment_plot","title":"enrichment_plot: create an enrichment plot for cumulative counts"},{"location":"user_guide/plotting/enrichment_plot/#overview","text":"In enrichment plots, the y-axis can be interpreted as \"how many samples are less or equal to the corresponding x-axis label.\"","title":"Overview"},{"location":"user_guide/plotting/enrichment_plot/#references","text":"-","title":"References"},{"location":"user_guide/plotting/enrichment_plot/#example-1-enrichment-plots-from-pandas-dataframes","text":"import pandas as pd s1 = [1.1, 1.5] s2 = [2.1, 1.8] s3 = [3.1, 2.1] s4 = [3.9, 2.5] data = [s1, s2, s3, s4] df = pd.DataFrame(data, columns=['X1', 'X2']) df X1 X2 0 1.1 1.5 1 2.1 1.8 2 3.1 2.1 3 3.9 2.5 Plotting the data where the categories are determined by the unique values in the label column label_col . The x and y values are simply the column names of the DataFrame that we want to plot. import matplotlib.pyplot as plt from mlxtend.plotting import enrichment_plot ax = enrichment_plot(df, legend_loc='upper left')","title":"Example 1 - Enrichment Plots from Pandas DataFrames"},{"location":"user_guide/plotting/enrichment_plot/#api","text":"enrichment_plot(df, colors='bgrkcy', markers=' ', linestyles='-', alpha=0.5, lw=2, where='post', grid=True, count_label='Count', xlim='auto', ylim='auto', invert_axes=False, legend_loc='best', ax=None) Plot stacked barplots Parameters df : pandas.DataFrame A pandas DataFrame where columns represent the different categories. colors: str (default: 'bgrcky') The colors of the bars. markers : str (default: ' ') Matplotlib markerstyles, e.g, 'sov' for square,circle, and triangle markers. linestyles : str (default: '-') Matplotlib linestyles, e.g., '-,--' to cycle normal and dashed lines. Note that the different linestyles need to be separated by commas. alpha : float (default: 0.5) Transparency level from 0.0 to 1.0. lw : int or float (default: 2) Linewidth parameter. where : {'post', 'pre', 'mid'} (default: 'post') Starting location of the steps. grid : bool (default: True ) Plots a grid if True. count_label : str (default: 'Count') Label for the \"Count\"-axis. xlim : 'auto' or array-like [min, max] (default: 'auto') Min and maximum position of the x-axis range. ylim : 'auto' or array-like [min, max] (default: 'auto') Min and maximum position of the y-axis range. invert_axes : bool (default: False) Plots count on the x-axis if True. legend_loc : str (default: 'best') Location of the plot legend {best, upper left, upper right, lower left, lower right} No legend if legend_loc=False ax : matplotlib axis, optional (default: None) Use this axis for plotting or make a new one otherwise Returns ax : matplotlib axis Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/plotting/enrichment_plot/","title":"API"},{"location":"user_guide/plotting/heatmap/","text":"heatmap: Create a heatmap in matplotlib Utility function for creating a heatmap via matplotlib from mlxtend.plotting import heatmap Overview A simple function that creates nice-looking heatmaps from NumPy arrays using matplotlib and the Viridis color palette by default. References - Example 1 -- Simple heatmap from mlxtend.plotting import heatmap import matplotlib.pyplot as plt import numpy as np import pandas as pd np.random.seed(123) some_array = np.random.random((20, 30)) heatmap(some_array, figsize=(20, 10)) plt.show() The cell values can be suppressed by setting cell_values=False : heatmap(some_array, figsize=(20, 10), cell_values=False) plt.show() Example 2 -- Correlation matrix as heatmap from mlxtend.plotting import heatmap import matplotlib.pyplot as plt import numpy as np import pandas as pd df = pd.read_csv('https://raw.githubusercontent.com/rasbt/' 'python-machine-learning-book-2nd-edition' '/master/code/ch10/housing.data.txt', header=None, sep='\\s+') df.columns = ['CRIM', 'ZN', 'INDUS', 'CHAS', 'NOX', 'RM', 'AGE', 'DIS', 'RAD', 'TAX', 'PTRATIO', 'B', 'LSTAT', 'MEDV'] df.head() .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } CRIM ZN INDUS CHAS NOX RM AGE DIS RAD TAX PTRATIO B LSTAT MEDV 0 0.00632 18.0 2.31 0 0.538 6.575 65.2 4.0900 1 296.0 15.3 396.90 4.98 24.0 1 0.02731 0.0 7.07 0 0.469 6.421 78.9 4.9671 2 242.0 17.8 396.90 9.14 21.6 2 0.02729 0.0 7.07 0 0.469 7.185 61.1 4.9671 2 242.0 17.8 392.83 4.03 34.7 3 0.03237 0.0 2.18 0 0.458 6.998 45.8 6.0622 3 222.0 18.7 394.63 2.94 33.4 4 0.06905 0.0 2.18 0 0.458 7.147 54.2 6.0622 3 222.0 18.7 396.90 5.33 36.2 from matplotlib import cm cols = ['LSTAT', 'INDUS', 'NOX', 'RM', 'MEDV'] corrmat = np.corrcoef(df[cols].values.T) fig, ax = heatmap(corrmat, column_names=cols, row_names=cols, cmap=cm.PiYG) # set colorbar cutoff at -1, 1 for im in ax.get_images(): im.set_clim(-1, 1) plt.show() API heatmap(matrix, hide_spines=False, hide_ticks=False, figsize=None, cmap=None, colorbar=True, row_names=None, column_names=None, column_name_rotation=45, cell_values=True, cell_fmt='.2f', cell_font_size=None, text_color_threshold=None) Plot a heatmap via matplotlib. Parameters conf_mat : array-like, shape = [n_rows, n_columns] And arbitrary 2D array. hide_spines : bool (default: False) Hides axis spines if True. hide_ticks : bool (default: False) Hides axis ticks if True figsize : tuple (default: (2.5, 2.5)) Height and width of the figure cmap : matplotlib colormap (default: None ) Uses matplotlib.pyplot.cm.viridis if None colorbar : bool (default: True) Shows a colorbar if True row_names : array-like, shape = [n_rows] (default: None) List of row names to be used as y-axis tick labels. column_names : array-like, shape = [n_columns] (default: None) List of column names to be used as x-axis tick labels. column_name_rotation : int (default: 45) Number of degrees for rotating column x-tick labels. cell_values : bool (default: True) Plots cell values if True. cell_fmt : string (default: '.2f') Format specification for cell values (if cell_values=True ) cell_font_size : int (default: None) Font size for cell values (if cell_values=True ) text_color_threshold : float (default: None) Threshold for the black/white text threshold of the text annotation. Default (None) tried to infer a good threshold automatically using np.max(normed_matrix) / 2 . Returns fig, ax : matplotlib.pyplot subplot objects Figure and axis elements of the subplot. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/plotting/heatmap/","title":"heatmap: Create a heatmap in matplotlib"},{"location":"user_guide/plotting/heatmap/#heatmap-create-a-heatmap-in-matplotlib","text":"Utility function for creating a heatmap via matplotlib from mlxtend.plotting import heatmap","title":"heatmap: Create a heatmap in matplotlib"},{"location":"user_guide/plotting/heatmap/#overview","text":"A simple function that creates nice-looking heatmaps from NumPy arrays using matplotlib and the Viridis color palette by default.","title":"Overview"},{"location":"user_guide/plotting/heatmap/#references","text":"-","title":"References"},{"location":"user_guide/plotting/heatmap/#example-1-simple-heatmap","text":"from mlxtend.plotting import heatmap import matplotlib.pyplot as plt import numpy as np import pandas as pd np.random.seed(123) some_array = np.random.random((20, 30)) heatmap(some_array, figsize=(20, 10)) plt.show() The cell values can be suppressed by setting cell_values=False : heatmap(some_array, figsize=(20, 10), cell_values=False) plt.show()","title":"Example 1 -- Simple heatmap"},{"location":"user_guide/plotting/heatmap/#example-2-correlation-matrix-as-heatmap","text":"from mlxtend.plotting import heatmap import matplotlib.pyplot as plt import numpy as np import pandas as pd df = pd.read_csv('https://raw.githubusercontent.com/rasbt/' 'python-machine-learning-book-2nd-edition' '/master/code/ch10/housing.data.txt', header=None, sep='\\s+') df.columns = ['CRIM', 'ZN', 'INDUS', 'CHAS', 'NOX', 'RM', 'AGE', 'DIS', 'RAD', 'TAX', 'PTRATIO', 'B', 'LSTAT', 'MEDV'] df.head() .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } CRIM ZN INDUS CHAS NOX RM AGE DIS RAD TAX PTRATIO B LSTAT MEDV 0 0.00632 18.0 2.31 0 0.538 6.575 65.2 4.0900 1 296.0 15.3 396.90 4.98 24.0 1 0.02731 0.0 7.07 0 0.469 6.421 78.9 4.9671 2 242.0 17.8 396.90 9.14 21.6 2 0.02729 0.0 7.07 0 0.469 7.185 61.1 4.9671 2 242.0 17.8 392.83 4.03 34.7 3 0.03237 0.0 2.18 0 0.458 6.998 45.8 6.0622 3 222.0 18.7 394.63 2.94 33.4 4 0.06905 0.0 2.18 0 0.458 7.147 54.2 6.0622 3 222.0 18.7 396.90 5.33 36.2 from matplotlib import cm cols = ['LSTAT', 'INDUS', 'NOX', 'RM', 'MEDV'] corrmat = np.corrcoef(df[cols].values.T) fig, ax = heatmap(corrmat, column_names=cols, row_names=cols, cmap=cm.PiYG) # set colorbar cutoff at -1, 1 for im in ax.get_images(): im.set_clim(-1, 1) plt.show()","title":"Example 2 -- Correlation matrix as heatmap"},{"location":"user_guide/plotting/heatmap/#api","text":"heatmap(matrix, hide_spines=False, hide_ticks=False, figsize=None, cmap=None, colorbar=True, row_names=None, column_names=None, column_name_rotation=45, cell_values=True, cell_fmt='.2f', cell_font_size=None, text_color_threshold=None) Plot a heatmap via matplotlib. Parameters conf_mat : array-like, shape = [n_rows, n_columns] And arbitrary 2D array. hide_spines : bool (default: False) Hides axis spines if True. hide_ticks : bool (default: False) Hides axis ticks if True figsize : tuple (default: (2.5, 2.5)) Height and width of the figure cmap : matplotlib colormap (default: None ) Uses matplotlib.pyplot.cm.viridis if None colorbar : bool (default: True) Shows a colorbar if True row_names : array-like, shape = [n_rows] (default: None) List of row names to be used as y-axis tick labels. column_names : array-like, shape = [n_columns] (default: None) List of column names to be used as x-axis tick labels. column_name_rotation : int (default: 45) Number of degrees for rotating column x-tick labels. cell_values : bool (default: True) Plots cell values if True. cell_fmt : string (default: '.2f') Format specification for cell values (if cell_values=True ) cell_font_size : int (default: None) Font size for cell values (if cell_values=True ) text_color_threshold : float (default: None) Threshold for the black/white text threshold of the text annotation. Default (None) tried to infer a good threshold automatically using np.max(normed_matrix) / 2 . Returns fig, ax : matplotlib.pyplot subplot objects Figure and axis elements of the subplot. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/plotting/heatmap/","title":"API"},{"location":"user_guide/plotting/plot_confusion_matrix/","text":"plot_confusion_matrix: Visualize confusion matrices Utility function for visualizing confusion matrices via matplotlib from mlxtend.plotting import plot_confusion_matrix Overview Confusion Matrix For more information on confusion matrices, please see mlxtend.evaluate.confusion_matrix . References - Example 1 - Binary from mlxtend.plotting import plot_confusion_matrix import matplotlib.pyplot as plt import numpy as np binary1 = np.array([[4, 1], [1, 2]]) fig, ax = plot_confusion_matrix(conf_mat=binary1) plt.show() binary2 = np.array([[21, 1], [3, 1]]) fig, ax = plot_confusion_matrix(conf_mat=binary2, figsize=(2, 2)) plt.show() Example 2 - Binary absolute and relative with colorbar binary = np.array([[4, 1], [1, 2]]) fig, ax = plot_confusion_matrix(conf_mat=binary, show_absolute=True, show_normed=True, colorbar=True) plt.show() Example 3 - Multiclass relative multiclass = np.array([[2, 1, 0, 0], [1, 2, 0, 0], [0, 0, 1, 0], [0, 0, 0, 1]]) fig, ax = plot_confusion_matrix(conf_mat=multiclass, colorbar=True, show_absolute=False, show_normed=True) plt.show() Example 4 - Add Class Names multiclass = np.array([[2, 1, 0, 0], [1, 2, 0, 0], [0, 0, 1, 0], [0, 0, 0, 1]]) class_names = ['class a', 'class b', 'class c', 'class d'] fig, ax = plot_confusion_matrix(conf_mat=multiclass, colorbar=True, show_absolute=False, show_normed=True, class_names=class_names) plt.show() Example 5 - Changing Color Maps and Font Color Matplotlib color maps can be chosen as alternative color map via the cmap argument. A list of colormaps can be found here: https://matplotlib.org/stable/tutorials/colors/colormaps.html multiclass = np.array([[2, 1, 0, 0], [1, 2, 0, 0], [0, 0, 1, 0], [0, 0, 0, 1]]) fig, ax = plot_confusion_matrix(conf_mat=multiclass, colorbar=True, cmap='summer') plt.show() As shown above, the font color threshold may not work for certain color maps. By default all values larger than 0.5 times the maximum cell value are converted to white, and everything equal or smaller than 0.5 times the maximum cell value are converted to black. If you want to change all values above to e.g., white, you can set the color threshold to a negative number. Or, if you want to make all the font colors black, choose ta threshold equal to or greater than 1. fig, ax = plot_confusion_matrix(conf_mat=multiclass, colorbar=True, fontcolor_threshold=1, cmap='summer') plt.show() Example 6 - Normalizing Colormaps to Highlight Off-Diagonals Suppose we have the following confusion matrix for a high-accuracy classifier: class_dict = {0: 'airplane', 1: 'automobile', 2: 'bird', 3: 'cat', 4: 'deer', 5: 'dog', 6: 'frog'} cmat = np.array([[972, 0, 1, 1, 1, 1, 3], [0, 1123, 3, 1, 0, 1, 2], [2, 0, 1025, 0, 0, 0, 1], [0, 0, 0, 1005, 0, 2, 0], [0, 1, 1, 0, 967, 0, 4], [0, 0, 0, 6, 0, 881, 3], [2, 3, 0, 1, 3, 4, 941]]) fig, ax = plot_confusion_matrix( conf_mat=cmat, class_names=class_dict.values(), ) It can be hard to notice the cells where the models makes mistakes. With a log-normalized colormap, these mistakes off the diagonal become easier to see at a glace: import matplotlib fig, ax = plot_confusion_matrix( conf_mat=cmat, class_names=class_dict.values(), norm_colormap=matplotlib.colors.LogNorm() ) API plot_confusion_matrix(conf_mat, hide_spines=False, hide_ticks=False, figsize=None, cmap=None, colorbar=False, show_absolute=True, show_normed=False, class_names=None, figure=None, axis=None, fontcolor_threshold=0.5) Plot a confusion matrix via matplotlib. Parameters conf_mat : array-like, shape = [n_classes, n_classes] Confusion matrix from evaluate.confusion matrix. hide_spines : bool (default: False) Hides axis spines if True. hide_ticks : bool (default: False) Hides axis ticks if True figsize : tuple (default: (2.5, 2.5)) Height and width of the figure cmap : matplotlib colormap (default: None ) Uses matplotlib.pyplot.cm.Blues if None colorbar : bool (default: False) Shows a colorbar if True show_absolute : bool (default: True) Shows absolute confusion matrix coefficients if True. At least one of show_absolute or show_normed must be True. show_normed : bool (default: False) Shows normed confusion matrix coefficients if True. The normed confusion matrix coefficients give the proportion of training examples per class that are assigned the correct label. At least one of show_absolute or show_normed must be True. class_names : array-like, shape = [n_classes] (default: None) List of class names. If not None , ticks will be set to these values. figure : None or Matplotlib figure (default: None) If None will create a new figure. axis : None or Matplotlib figure axis (default: None) If None will create a new axis. fontcolor_threshold : Float (default: 0.5) Sets a threshold for choosing black and white font colors for the cells. By default all values larger than 0.5 times the maximum cell value are converted to white, and everything equal or smaller than 0.5 times the maximum cell value are converted to black. Returns fig, ax : matplotlib.pyplot subplot objects Figure and axis elements of the subplot. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/plotting/plot_confusion_matrix/","title":"plot_confusion_matrix: Visualize confusion matrices"},{"location":"user_guide/plotting/plot_confusion_matrix/#plot_confusion_matrix-visualize-confusion-matrices","text":"Utility function for visualizing confusion matrices via matplotlib from mlxtend.plotting import plot_confusion_matrix","title":"plot_confusion_matrix: Visualize confusion matrices"},{"location":"user_guide/plotting/plot_confusion_matrix/#overview","text":"","title":"Overview"},{"location":"user_guide/plotting/plot_confusion_matrix/#confusion-matrix","text":"For more information on confusion matrices, please see mlxtend.evaluate.confusion_matrix .","title":"Confusion Matrix"},{"location":"user_guide/plotting/plot_confusion_matrix/#references","text":"-","title":"References"},{"location":"user_guide/plotting/plot_confusion_matrix/#example-1-binary","text":"from mlxtend.plotting import plot_confusion_matrix import matplotlib.pyplot as plt import numpy as np binary1 = np.array([[4, 1], [1, 2]]) fig, ax = plot_confusion_matrix(conf_mat=binary1) plt.show() binary2 = np.array([[21, 1], [3, 1]]) fig, ax = plot_confusion_matrix(conf_mat=binary2, figsize=(2, 2)) plt.show()","title":"Example 1 - Binary"},{"location":"user_guide/plotting/plot_confusion_matrix/#example-2-binary-absolute-and-relative-with-colorbar","text":"binary = np.array([[4, 1], [1, 2]]) fig, ax = plot_confusion_matrix(conf_mat=binary, show_absolute=True, show_normed=True, colorbar=True) plt.show()","title":"Example 2 - Binary absolute and relative with colorbar"},{"location":"user_guide/plotting/plot_confusion_matrix/#example-3-multiclass-relative","text":"multiclass = np.array([[2, 1, 0, 0], [1, 2, 0, 0], [0, 0, 1, 0], [0, 0, 0, 1]]) fig, ax = plot_confusion_matrix(conf_mat=multiclass, colorbar=True, show_absolute=False, show_normed=True) plt.show()","title":"Example 3 - Multiclass relative"},{"location":"user_guide/plotting/plot_confusion_matrix/#example-4-add-class-names","text":"multiclass = np.array([[2, 1, 0, 0], [1, 2, 0, 0], [0, 0, 1, 0], [0, 0, 0, 1]]) class_names = ['class a', 'class b', 'class c', 'class d'] fig, ax = plot_confusion_matrix(conf_mat=multiclass, colorbar=True, show_absolute=False, show_normed=True, class_names=class_names) plt.show()","title":"Example 4 - Add Class Names"},{"location":"user_guide/plotting/plot_confusion_matrix/#example-5-changing-color-maps-and-font-color","text":"Matplotlib color maps can be chosen as alternative color map via the cmap argument. A list of colormaps can be found here: https://matplotlib.org/stable/tutorials/colors/colormaps.html multiclass = np.array([[2, 1, 0, 0], [1, 2, 0, 0], [0, 0, 1, 0], [0, 0, 0, 1]]) fig, ax = plot_confusion_matrix(conf_mat=multiclass, colorbar=True, cmap='summer') plt.show() As shown above, the font color threshold may not work for certain color maps. By default all values larger than 0.5 times the maximum cell value are converted to white, and everything equal or smaller than 0.5 times the maximum cell value are converted to black. If you want to change all values above to e.g., white, you can set the color threshold to a negative number. Or, if you want to make all the font colors black, choose ta threshold equal to or greater than 1. fig, ax = plot_confusion_matrix(conf_mat=multiclass, colorbar=True, fontcolor_threshold=1, cmap='summer') plt.show()","title":"Example 5 - Changing Color Maps and Font Color"},{"location":"user_guide/plotting/plot_confusion_matrix/#example-6-normalizing-colormaps-to-highlight-off-diagonals","text":"Suppose we have the following confusion matrix for a high-accuracy classifier: class_dict = {0: 'airplane', 1: 'automobile', 2: 'bird', 3: 'cat', 4: 'deer', 5: 'dog', 6: 'frog'} cmat = np.array([[972, 0, 1, 1, 1, 1, 3], [0, 1123, 3, 1, 0, 1, 2], [2, 0, 1025, 0, 0, 0, 1], [0, 0, 0, 1005, 0, 2, 0], [0, 1, 1, 0, 967, 0, 4], [0, 0, 0, 6, 0, 881, 3], [2, 3, 0, 1, 3, 4, 941]]) fig, ax = plot_confusion_matrix( conf_mat=cmat, class_names=class_dict.values(), ) It can be hard to notice the cells where the models makes mistakes. With a log-normalized colormap, these mistakes off the diagonal become easier to see at a glace: import matplotlib fig, ax = plot_confusion_matrix( conf_mat=cmat, class_names=class_dict.values(), norm_colormap=matplotlib.colors.LogNorm() )","title":"Example 6 - Normalizing Colormaps to Highlight Off-Diagonals"},{"location":"user_guide/plotting/plot_confusion_matrix/#api","text":"plot_confusion_matrix(conf_mat, hide_spines=False, hide_ticks=False, figsize=None, cmap=None, colorbar=False, show_absolute=True, show_normed=False, class_names=None, figure=None, axis=None, fontcolor_threshold=0.5) Plot a confusion matrix via matplotlib. Parameters conf_mat : array-like, shape = [n_classes, n_classes] Confusion matrix from evaluate.confusion matrix. hide_spines : bool (default: False) Hides axis spines if True. hide_ticks : bool (default: False) Hides axis ticks if True figsize : tuple (default: (2.5, 2.5)) Height and width of the figure cmap : matplotlib colormap (default: None ) Uses matplotlib.pyplot.cm.Blues if None colorbar : bool (default: False) Shows a colorbar if True show_absolute : bool (default: True) Shows absolute confusion matrix coefficients if True. At least one of show_absolute or show_normed must be True. show_normed : bool (default: False) Shows normed confusion matrix coefficients if True. The normed confusion matrix coefficients give the proportion of training examples per class that are assigned the correct label. At least one of show_absolute or show_normed must be True. class_names : array-like, shape = [n_classes] (default: None) List of class names. If not None , ticks will be set to these values. figure : None or Matplotlib figure (default: None) If None will create a new figure. axis : None or Matplotlib figure axis (default: None) If None will create a new axis. fontcolor_threshold : Float (default: 0.5) Sets a threshold for choosing black and white font colors for the cells. By default all values larger than 0.5 times the maximum cell value are converted to white, and everything equal or smaller than 0.5 times the maximum cell value are converted to black. Returns fig, ax : matplotlib.pyplot subplot objects Figure and axis elements of the subplot. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/plotting/plot_confusion_matrix/","title":"API"},{"location":"user_guide/plotting/plot_decision_regions/","text":"plot_decision_regions: Visualize the decision regions of a classifier A function for plotting decision regions of classifiers in 1 or 2 dimensions. from mlxtend.plotting import plot_decision_regions References Example 1 - Decision regions in 2D from mlxtend.plotting import plot_decision_regions import matplotlib.pyplot as plt from sklearn import datasets from sklearn.svm import SVC # Loading some example data iris = datasets.load_iris() X = iris.data[:, [0, 2]] y = iris.target # Training a classifier svm = SVC(C=0.5, kernel='linear') svm.fit(X, y) # Plotting decision regions plot_decision_regions(X, y, clf=svm, legend=2) # Adding axes annotations plt.xlabel('sepal length [cm]') plt.ylabel('petal length [cm]') plt.title('SVM on Iris') plt.show() Example 2 - Decision regions in 1D from mlxtend.plotting import plot_decision_regions import matplotlib.pyplot as plt from sklearn import datasets from sklearn.svm import SVC # Loading some example data iris = datasets.load_iris() X = iris.data[:, 2] X = X[:, None] y = iris.target # Training a classifier svm = SVC(C=0.5, kernel='linear') svm.fit(X, y) # Plotting decision regions plot_decision_regions(X, y, clf=svm, legend=2) # Adding axes annotations plt.xlabel('sepal length [cm]') plt.title('SVM on Iris') plt.show() Example 3 - Decision Region Grids from sklearn.linear_model import LogisticRegression from sklearn.naive_bayes import GaussianNB from sklearn.ensemble import RandomForestClassifier from sklearn.svm import SVC from sklearn import datasets import numpy as np # Initializing Classifiers clf1 = LogisticRegression(random_state=1, solver='newton-cg', multi_class='multinomial') clf2 = RandomForestClassifier(random_state=1, n_estimators=100) clf3 = GaussianNB() clf4 = SVC(gamma='auto') # Loading some example data iris = datasets.load_iris() X = iris.data[:, [0,2]] y = iris.target import matplotlib.pyplot as plt from mlxtend.plotting import plot_decision_regions import matplotlib.gridspec as gridspec import itertools gs = gridspec.GridSpec(2, 2) fig = plt.figure(figsize=(10,8)) labels = ['Logistic Regression', 'Random Forest', 'Naive Bayes', 'SVM'] for clf, lab, grd in zip([clf1, clf2, clf3, clf4], labels, itertools.product([0, 1], repeat=2)): clf.fit(X, y) ax = plt.subplot(gs[grd[0], grd[1]]) fig = plot_decision_regions(X=X, y=y, clf=clf, legend=2) plt.title(lab) plt.show() Example 4 - Highlighting Test Data Points from mlxtend.plotting import plot_decision_regions from mlxtend.preprocessing import shuffle_arrays_unison import matplotlib.pyplot as plt from sklearn import datasets from sklearn.svm import SVC # Loading some example data iris = datasets.load_iris() X, y = iris.data[:, [0,2]], iris.target X, y = shuffle_arrays_unison(arrays=[X, y], random_seed=3) X_train, y_train = X[:100], y[:100] X_test, y_test = X[100:], y[100:] # Training a classifier svm = SVC(C=0.5, kernel='linear') svm.fit(X_train, y_train) # Plotting decision regions plot_decision_regions(X, y, clf=svm, legend=2, X_highlight=X_test) # Adding axes annotations plt.xlabel('sepal length [cm]') plt.ylabel('petal length [cm]') plt.title('SVM on Iris') plt.show() Example 5 - Evaluating Classifier Behavior on Non-Linear Problems from sklearn.linear_model import LogisticRegression from sklearn.naive_bayes import GaussianNB from sklearn.ensemble import RandomForestClassifier from sklearn.svm import SVC # Initializing Classifiers clf1 = LogisticRegression(random_state=1, solver='lbfgs') clf2 = RandomForestClassifier(n_estimators=100, random_state=1) clf3 = GaussianNB() clf4 = SVC(gamma='auto') # Loading Plotting Utilities import matplotlib.pyplot as plt import matplotlib.gridspec as gridspec import itertools from mlxtend.plotting import plot_decision_regions import numpy as np XOR xx, yy = np.meshgrid(np.linspace(-3, 3, 50), np.linspace(-3, 3, 50)) rng = np.random.RandomState(0) X = rng.randn(300, 2) y = np.array(np.logical_xor(X[:, 0] > 0, X[:, 1] > 0), dtype=int) gs = gridspec.GridSpec(2, 2) fig = plt.figure(figsize=(10,8)) labels = ['Logistic Regression', 'Random Forest', 'Naive Bayes', 'SVM'] for clf, lab, grd in zip([clf1, clf2, clf3, clf4], labels, itertools.product([0, 1], repeat=2)): clf.fit(X, y) ax = plt.subplot(gs[grd[0], grd[1]]) fig = plot_decision_regions(X=X, y=y, clf=clf, legend=2) plt.title(lab) plt.show() Half-Moons from sklearn.datasets import make_moons X, y = make_moons(n_samples=100, random_state=123) gs = gridspec.GridSpec(2, 2) fig = plt.figure(figsize=(10,8)) labels = ['Logistic Regression', 'Random Forest', 'Naive Bayes', 'SVM'] for clf, lab, grd in zip([clf1, clf2, clf3, clf4], labels, itertools.product([0, 1], repeat=2)): clf.fit(X, y) ax = plt.subplot(gs[grd[0], grd[1]]) fig = plot_decision_regions(X=X, y=y, clf=clf, legend=2) plt.title(lab) plt.show() Concentric Circles from sklearn.datasets import make_circles X, y = make_circles(n_samples=1000, random_state=123, noise=0.1, factor=0.2) gs = gridspec.GridSpec(2, 2) fig = plt.figure(figsize=(10,8)) labels = ['Logistic Regression', 'Random Forest', 'Naive Bayes', 'SVM'] for clf, lab, grd in zip([clf1, clf2, clf3, clf4], labels, itertools.product([0, 1], repeat=2)): clf.fit(X, y) ax = plt.subplot(gs[grd[0], grd[1]]) fig = plot_decision_regions(X=X, y=y, clf=clf, legend=2) plt.title(lab) plt.show() Example 6 - Working with existing axes objects using subplots import matplotlib.pyplot as plt from mlxtend.plotting import plot_decision_regions from sklearn.linear_model import LogisticRegression from sklearn.naive_bayes import GaussianNB from sklearn import datasets import numpy as np # Loading some example data iris = datasets.load_iris() X = iris.data[:, 2] X = X[:, None] y = iris.target # Initializing and fitting classifiers clf1 = LogisticRegression(random_state=1, solver='lbfgs', multi_class='multinomial') clf2 = GaussianNB() clf1.fit(X, y) clf2.fit(X, y) fig, axes = plt.subplots(1, 2, figsize=(10, 3)) fig = plot_decision_regions(X=X, y=y, clf=clf1, ax=axes[0], legend=2) fig = plot_decision_regions(X=X, y=y, clf=clf2, ax=axes[1], legend=1) plt.show() Example 7 - Decision regions with more than two training features from mlxtend.plotting import plot_decision_regions import matplotlib.pyplot as plt from sklearn import datasets from sklearn.svm import SVC # Loading some example data X, y = datasets.make_blobs(n_samples=600, n_features=3, centers=[[2, 2, -2],[-2, -2, 2]], cluster_std=[2, 2], random_state=2) # Training a classifier svm = SVC(gamma='auto') svm.fit(X, y) # Plotting decision regions fig, ax = plt.subplots() # Decision region for feature 3 = 1.5 value = 1.5 # Plot training sample with feature 3 = 1.5 +/- 0.75 width = 0.75 plot_decision_regions(X, y, clf=svm, filler_feature_values={2: value}, filler_feature_ranges={2: width}, legend=2, ax=ax) ax.set_xlabel('Feature 1') ax.set_ylabel('Feature 2') ax.set_title('Feature 3 = {}'.format(value)) # Adding axes annotations fig.suptitle('SVM on make_blobs') plt.show() Example 8 - Grid of decision region slices from mlxtend.plotting import plot_decision_regions import matplotlib.pyplot as plt from sklearn import datasets from sklearn.svm import SVC # Loading some example data X, y = datasets.make_blobs(n_samples=500, n_features=3, centers=[[2, 2, -2],[-2, -2, 2]], cluster_std=[2, 2], random_state=2) # Training a classifier svm = SVC(gamma='auto') svm.fit(X, y) # Plotting decision regions fig, axarr = plt.subplots(2, 2, figsize=(10,8), sharex=True, sharey=True) values = [-4.0, -1.0, 1.0, 4.0] width = 0.75 for value, ax in zip(values, axarr.flat): plot_decision_regions(X, y, clf=svm, filler_feature_values={2: value}, filler_feature_ranges={2: width}, legend=2, ax=ax) ax.set_xlabel('Feature 1') ax.set_ylabel('Feature 2') ax.set_title('Feature 3 = {}'.format(value)) # Adding axes annotations fig.suptitle('SVM on make_blobs') plt.show() Example 9 - Customizing the plotting style from mlxtend.plotting import plot_decision_regions from mlxtend.preprocessing import shuffle_arrays_unison import matplotlib.pyplot as plt from sklearn import datasets from sklearn.svm import SVC # Loading some example data iris = datasets.load_iris() X = iris.data[:, [0, 2]] y = iris.target X, y = shuffle_arrays_unison(arrays=[X, y], random_seed=3) X_train, y_train = X[:100], y[:100] X_test, y_test = X[100:], y[100:] # Training a classifier svm = SVC(C=0.5, kernel='linear') svm.fit(X_train, y_train) # Specify keyword arguments to be passed to underlying plotting functions scatter_kwargs = {'s': 120, 'edgecolor': None, 'alpha': 0.7} contourf_kwargs = {'alpha': 0.2} scatter_highlight_kwargs = {'s': 120, 'label': 'Test data', 'alpha': 0.7} # Plotting decision regions plot_decision_regions(X, y, clf=svm, legend=2, X_highlight=X_test, scatter_kwargs=scatter_kwargs, contourf_kwargs=contourf_kwargs, scatter_highlight_kwargs=scatter_highlight_kwargs) # Adding axes annotations plt.xlabel('sepal length [cm]') plt.ylabel('petal length [cm]') plt.title('SVM on Iris') plt.show() Example 10 - Providing your own legend labels Custom legend labels can be provided by returning the axis object(s) from the plot_decision_region function and then getting the handles and labels of the legend. Custom handles (i.e., labels) can then be provided via ax.legend ax = plot_decision_regions(X, y, clf=svm, legend=0) handles, labels = ax.get_legend_handles_labels() ax.legend(handles, ['class 0', 'class 1', 'class 2'], framealpha=0.3, scatterpoints=1) An example is shown below. from mlxtend.plotting import plot_decision_regions import matplotlib.pyplot as plt from sklearn import datasets from sklearn.svm import SVC # Loading some example data iris = datasets.load_iris() X = iris.data[:, [0, 2]] y = iris.target # Training a classifier svm = SVC(C=0.5, kernel='linear') svm.fit(X, y) # Plotting decision regions ax = plot_decision_regions(X, y, clf=svm, legend=0) # Adding axes annotations plt.xlabel('sepal length [cm]') plt.ylabel('petal length [cm]') plt.title('SVM on Iris') handles, labels = ax.get_legend_handles_labels() ax.legend(handles, ['class square', 'class triangle', 'class circle'], framealpha=0.3, scatterpoints=1) plt.show() Example 11 - Plots with different zoom factors from mlxtend.plotting import plot_decision_regions import matplotlib.pyplot as plt from sklearn import datasets from sklearn.svm import SVC # Loading some example data iris = datasets.load_iris() X = iris.data[:, [0, 2]] y = iris.target # Training a classifier svm = SVC(C=0.5, kernel='linear') svm.fit(X, y) #sk-container-id-1 {color: black;background-color: white;}#sk-container-id-1 pre{padding: 0;}#sk-container-id-1 div.sk-toggleable {background-color: white;}#sk-container-id-1 label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.3em;box-sizing: border-box;text-align: center;}#sk-container-id-1 label.sk-toggleable__label-arrow:before {content: \"\u25b8\";float: left;margin-right: 0.25em;color: #696969;}#sk-container-id-1 label.sk-toggleable__label-arrow:hover:before {color: black;}#sk-container-id-1 div.sk-estimator:hover label.sk-toggleable__label-arrow:before {color: black;}#sk-container-id-1 div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}#sk-container-id-1 div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}#sk-container-id-1 input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}#sk-container-id-1 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {content: \"\u25be\";}#sk-container-id-1 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}#sk-container-id-1 div.sk-estimator {font-family: monospace;background-color: #f0f8ff;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;margin-bottom: 0.5em;}#sk-container-id-1 div.sk-estimator:hover {background-color: #d4ebff;}#sk-container-id-1 div.sk-parallel-item::after {content: \"\";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}#sk-container-id-1 div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 div.sk-serial::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: 0;}#sk-container-id-1 div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;padding-right: 0.2em;padding-left: 0.2em;position: relative;}#sk-container-id-1 div.sk-item {position: relative;z-index: 1;}#sk-container-id-1 div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;position: relative;}#sk-container-id-1 div.sk-item::before, #sk-container-id-1 div.sk-parallel-item::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: -1;}#sk-container-id-1 div.sk-parallel-item {display: flex;flex-direction: column;z-index: 1;position: relative;background-color: white;}#sk-container-id-1 div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}#sk-container-id-1 div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}#sk-container-id-1 div.sk-parallel-item:only-child::after {width: 0;}#sk-container-id-1 div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0 0.4em 0.5em 0.4em;box-sizing: border-box;padding-bottom: 0.4em;background-color: white;}#sk-container-id-1 div.sk-label label {font-family: monospace;font-weight: bold;display: inline-block;line-height: 1.2em;}#sk-container-id-1 div.sk-label-container {text-align: center;}#sk-container-id-1 div.sk-container {/* jupyter's `normalize.less` sets `[hidden] { display: none; }` but bootstrap.min.css set `[hidden] { display: none !important; }` so we also need the `!important` here to be able to override the default hidden behavior on the sphinx rendered scikit-learn.org. See: https://github.com/scikit-learn/scikit-learn/issues/21755 */display: inline-block !important;position: relative;}#sk-container-id-1 div.sk-text-repr-fallback {display: none;} SVC(C=0.5, kernel='linear') In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org. SVC SVC(C=0.5, kernel='linear') Default Zoom Factor plot_decision_regions(X, y, clf=svm, zoom_factor=1.) plt.show() Zooming out plot_decision_regions(X, y, clf=svm, zoom_factor=0.1) plt.show() Zooming in Note that while zooming in (by choosing a zoom_factor > 1.0) the plots are still created such that all data points are shown in the plot. plot_decision_regions(X, y, clf=svm, zoom_factor=2.0) plt.show() Cropping the axes In order to zoom in further, which means that some training examples won't be shown, you can simply crop the axes as shown below: plot_decision_regions(X, y, clf=svm, zoom_factor=2.0) plt.xlim(5, 6) plt.ylim(2, 5) plt.show() Example 12 - Using classifiers that expect onehot-encoded outputs (Keras) Most objects for classification that mimick the scikit-learn estimator API should be compatible with the plot_decision_regions function. However, if the classification model (e.g., a typical Keras model) output onehot-encoded predictions, we have to use an additional trick. I.e., for onehot encoded outputs, we need to wrap the Keras model into a class that converts these onehot encoded variables into integers. Such a wrapper class can be as simple as the following: class Onehot2Int(object): def __init__(self, model): self.model = model def predict(self, X): y_pred = self.model.predict(X) return np.argmax(y_pred, axis=1) The example below illustrates how the Onehot2Int class can be used with a Keras model that outputs onehot encoded labels: import keras from keras.models import Sequential from keras.layers import Dense import matplotlib.pyplot as plt import numpy as np from mlxtend.data import iris_data from mlxtend.preprocessing import standardize from mlxtend.plotting import plot_decision_regions from keras.utils import to_categorical X, y = iris_data() X = X[:, [2, 3]] X = standardize(X) # OneHot encoding y_onehot = to_categorical(y) # Create the model np.random.seed(123) model = Sequential() model.add(Dense(8, input_shape=(2,), activation='relu', kernel_initializer='he_uniform')) model.add(Dense(4, activation='relu', kernel_initializer='he_uniform')) model.add(Dense(3, activation='softmax')) # Configure the model and start training model.compile(loss=\"categorical_crossentropy\", optimizer=keras.optimizers.Adam(lr=0.005), metrics=['accuracy']) history = model.fit(X, y_onehot, epochs=10, batch_size=5, verbose=1, validation_split=0.1) Epoch 1/10 1/27 [>.............................] - ETA: 3s - loss: 1.2769 - accuracy: 0.4000 /Users/sebastianraschka/miniforge3/envs/mlxtend/lib/python3.8/site-packages/keras/optimizers/optimizer_v2/adam.py:117: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead. super().__init__(name, **kwargs) 2023-03-28 17:48:13.901264: W tensorflow/tsl/platform/profile_utils/cpu_utils.cc:128] Failed to get CPU frequency: 0 Hz 27/27 [==============================] - 0s 3ms/step - loss: 0.9526 - accuracy: 0.4222 - val_loss: 1.2656 - val_accuracy: 0.0000e+00 Epoch 2/10 27/27 [==============================] - 0s 834us/step - loss: 0.7062 - accuracy: 0.6741 - val_loss: 1.0939 - val_accuracy: 0.0000e+00 Epoch 3/10 27/27 [==============================] - 0s 808us/step - loss: 0.6461 - accuracy: 0.7111 - val_loss: 1.0705 - val_accuracy: 0.0667 Epoch 4/10 27/27 [==============================] - 0s 767us/step - loss: 0.6145 - accuracy: 0.7185 - val_loss: 1.0518 - val_accuracy: 0.0000e+00 Epoch 5/10 27/27 [==============================] - 0s 746us/step - loss: 0.5877 - accuracy: 0.7185 - val_loss: 1.0470 - val_accuracy: 0.0000e+00 Epoch 6/10 27/27 [==============================] - 0s 740us/step - loss: 0.5496 - accuracy: 0.7333 - val_loss: 1.0275 - val_accuracy: 0.0000e+00 Epoch 7/10 27/27 [==============================] - 0s 734us/step - loss: 0.4985 - accuracy: 0.7333 - val_loss: 1.0131 - val_accuracy: 0.0000e+00 Epoch 8/10 27/27 [==============================] - 0s 739us/step - loss: 0.4365 - accuracy: 0.7333 - val_loss: 0.9634 - val_accuracy: 0.0000e+00 Epoch 9/10 27/27 [==============================] - 0s 729us/step - loss: 0.3875 - accuracy: 0.7333 - val_loss: 0.9442 - val_accuracy: 0.0000e+00 Epoch 10/10 27/27 [==============================] - 0s 764us/step - loss: 0.3402 - accuracy: 0.7407 - val_loss: 0.8565 - val_accuracy: 0.0000e+00 # Wrap keras model model_no_ohe = Onehot2Int(model) # Plot decision boundary plot_decision_regions(X, y, clf=model_no_ohe) plt.show() 9600/9600 [==============================] - 3s 289us/step API plot_decision_regions(X, y, clf, feature_index=None, filler_feature_values=None, filler_feature_ranges=None, ax=None, X_highlight=None, zoom_factor=1.0, legend=1, hide_spines=True, markers='s^oxv<>', colors='#1f77b4,#ff7f0e,#3ca02c,#d62728,#9467bd,#8c564b,#e377c2,#7f7f7f,#bcbd22,#17becf', scatter_kwargs=None, contourf_kwargs=None, contour_kwargs=None, scatter_highlight_kwargs=None, n_jobs=None) Plot decision regions of a classifier. Please note that this functions assumes that class labels are labeled consecutively, e.g,. 0, 1, 2, 3, 4, and 5. If you have class labels with integer labels > 4, you may want to provide additional colors and/or markers as `colors` and `markers` arguments. See https://matplotlib.org/examples/color/named_colors.html for more information. Parameters X : array-like, shape = [n_samples, n_features] Feature Matrix. y : array-like, shape = [n_samples] True class labels. clf : Classifier object. Must have a .predict method. feature_index : array-like (default: (0,) for 1D, (0, 1) otherwise) Feature indices to use for plotting. The first index in feature_index will be on the x-axis, the second index will be on the y-axis. filler_feature_values : dict (default: None) Only needed for number features > 2. Dictionary of feature index-value pairs for the features not being plotted. filler_feature_ranges : dict (default: None) Only needed for number features > 2. Dictionary of feature index-value pairs for the features not being plotted. Will use the ranges provided to select training samples for plotting. ax : matplotlib.axes.Axes (default: None) An existing matplotlib Axes. Creates one if ax=None. X_highlight : array-like, shape = [n_samples, n_features] (default: None) An array with data points that are used to highlight samples in X . zoom_factor : float (default: 1.0) Controls the scale of the x- and y-axis of the decision plot. hide_spines : bool (default: True) Hide axis spines if True. legend : int (default: 1) Integer to specify the legend location. No legend if legend is 0. markers : str (default: 's^oxv<>') Scatterplot markers. colors : str (default: 'red,blue,limegreen,gray,cyan') Comma separated list of colors. scatter_kwargs : dict (default: None) Keyword arguments for underlying matplotlib scatter function. contourf_kwargs : dict (default: None) Keyword arguments for underlying matplotlib contourf function. contour_kwargs : dict (default: None) Keyword arguments for underlying matplotlib contour function (which draws the lines between decision regions). scatter_highlight_kwargs : dict (default: None) Keyword arguments for underlying matplotlib scatter function. n_jobs : int or None, optional (default=None) The number of CPUs to use to do the computation using Python's multiprocessing library. None means 1. -1 means using all processors. New in v0.22.0. Returns ax : matplotlib.axes.Axes object Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/plotting/plot_decision_regions/","title":"plot_decision_regions: Visualize the decision regions of a classifier"},{"location":"user_guide/plotting/plot_decision_regions/#plot_decision_regions-visualize-the-decision-regions-of-a-classifier","text":"A function for plotting decision regions of classifiers in 1 or 2 dimensions. from mlxtend.plotting import plot_decision_regions","title":"plot_decision_regions: Visualize the decision regions of a classifier"},{"location":"user_guide/plotting/plot_decision_regions/#references","text":"","title":"References"},{"location":"user_guide/plotting/plot_decision_regions/#example-1-decision-regions-in-2d","text":"from mlxtend.plotting import plot_decision_regions import matplotlib.pyplot as plt from sklearn import datasets from sklearn.svm import SVC # Loading some example data iris = datasets.load_iris() X = iris.data[:, [0, 2]] y = iris.target # Training a classifier svm = SVC(C=0.5, kernel='linear') svm.fit(X, y) # Plotting decision regions plot_decision_regions(X, y, clf=svm, legend=2) # Adding axes annotations plt.xlabel('sepal length [cm]') plt.ylabel('petal length [cm]') plt.title('SVM on Iris') plt.show()","title":"Example 1 - Decision regions in 2D"},{"location":"user_guide/plotting/plot_decision_regions/#example-2-decision-regions-in-1d","text":"from mlxtend.plotting import plot_decision_regions import matplotlib.pyplot as plt from sklearn import datasets from sklearn.svm import SVC # Loading some example data iris = datasets.load_iris() X = iris.data[:, 2] X = X[:, None] y = iris.target # Training a classifier svm = SVC(C=0.5, kernel='linear') svm.fit(X, y) # Plotting decision regions plot_decision_regions(X, y, clf=svm, legend=2) # Adding axes annotations plt.xlabel('sepal length [cm]') plt.title('SVM on Iris') plt.show()","title":"Example 2 - Decision regions in 1D"},{"location":"user_guide/plotting/plot_decision_regions/#example-3-decision-region-grids","text":"from sklearn.linear_model import LogisticRegression from sklearn.naive_bayes import GaussianNB from sklearn.ensemble import RandomForestClassifier from sklearn.svm import SVC from sklearn import datasets import numpy as np # Initializing Classifiers clf1 = LogisticRegression(random_state=1, solver='newton-cg', multi_class='multinomial') clf2 = RandomForestClassifier(random_state=1, n_estimators=100) clf3 = GaussianNB() clf4 = SVC(gamma='auto') # Loading some example data iris = datasets.load_iris() X = iris.data[:, [0,2]] y = iris.target import matplotlib.pyplot as plt from mlxtend.plotting import plot_decision_regions import matplotlib.gridspec as gridspec import itertools gs = gridspec.GridSpec(2, 2) fig = plt.figure(figsize=(10,8)) labels = ['Logistic Regression', 'Random Forest', 'Naive Bayes', 'SVM'] for clf, lab, grd in zip([clf1, clf2, clf3, clf4], labels, itertools.product([0, 1], repeat=2)): clf.fit(X, y) ax = plt.subplot(gs[grd[0], grd[1]]) fig = plot_decision_regions(X=X, y=y, clf=clf, legend=2) plt.title(lab) plt.show()","title":"Example 3 - Decision Region Grids"},{"location":"user_guide/plotting/plot_decision_regions/#example-4-highlighting-test-data-points","text":"from mlxtend.plotting import plot_decision_regions from mlxtend.preprocessing import shuffle_arrays_unison import matplotlib.pyplot as plt from sklearn import datasets from sklearn.svm import SVC # Loading some example data iris = datasets.load_iris() X, y = iris.data[:, [0,2]], iris.target X, y = shuffle_arrays_unison(arrays=[X, y], random_seed=3) X_train, y_train = X[:100], y[:100] X_test, y_test = X[100:], y[100:] # Training a classifier svm = SVC(C=0.5, kernel='linear') svm.fit(X_train, y_train) # Plotting decision regions plot_decision_regions(X, y, clf=svm, legend=2, X_highlight=X_test) # Adding axes annotations plt.xlabel('sepal length [cm]') plt.ylabel('petal length [cm]') plt.title('SVM on Iris') plt.show()","title":"Example 4 - Highlighting Test Data Points"},{"location":"user_guide/plotting/plot_decision_regions/#example-5-evaluating-classifier-behavior-on-non-linear-problems","text":"from sklearn.linear_model import LogisticRegression from sklearn.naive_bayes import GaussianNB from sklearn.ensemble import RandomForestClassifier from sklearn.svm import SVC # Initializing Classifiers clf1 = LogisticRegression(random_state=1, solver='lbfgs') clf2 = RandomForestClassifier(n_estimators=100, random_state=1) clf3 = GaussianNB() clf4 = SVC(gamma='auto') # Loading Plotting Utilities import matplotlib.pyplot as plt import matplotlib.gridspec as gridspec import itertools from mlxtend.plotting import plot_decision_regions import numpy as np","title":"Example 5 - Evaluating Classifier Behavior on Non-Linear Problems"},{"location":"user_guide/plotting/plot_decision_regions/#xor","text":"xx, yy = np.meshgrid(np.linspace(-3, 3, 50), np.linspace(-3, 3, 50)) rng = np.random.RandomState(0) X = rng.randn(300, 2) y = np.array(np.logical_xor(X[:, 0] > 0, X[:, 1] > 0), dtype=int) gs = gridspec.GridSpec(2, 2) fig = plt.figure(figsize=(10,8)) labels = ['Logistic Regression', 'Random Forest', 'Naive Bayes', 'SVM'] for clf, lab, grd in zip([clf1, clf2, clf3, clf4], labels, itertools.product([0, 1], repeat=2)): clf.fit(X, y) ax = plt.subplot(gs[grd[0], grd[1]]) fig = plot_decision_regions(X=X, y=y, clf=clf, legend=2) plt.title(lab) plt.show()","title":"XOR"},{"location":"user_guide/plotting/plot_decision_regions/#half-moons","text":"from sklearn.datasets import make_moons X, y = make_moons(n_samples=100, random_state=123) gs = gridspec.GridSpec(2, 2) fig = plt.figure(figsize=(10,8)) labels = ['Logistic Regression', 'Random Forest', 'Naive Bayes', 'SVM'] for clf, lab, grd in zip([clf1, clf2, clf3, clf4], labels, itertools.product([0, 1], repeat=2)): clf.fit(X, y) ax = plt.subplot(gs[grd[0], grd[1]]) fig = plot_decision_regions(X=X, y=y, clf=clf, legend=2) plt.title(lab) plt.show()","title":"Half-Moons"},{"location":"user_guide/plotting/plot_decision_regions/#concentric-circles","text":"from sklearn.datasets import make_circles X, y = make_circles(n_samples=1000, random_state=123, noise=0.1, factor=0.2) gs = gridspec.GridSpec(2, 2) fig = plt.figure(figsize=(10,8)) labels = ['Logistic Regression', 'Random Forest', 'Naive Bayes', 'SVM'] for clf, lab, grd in zip([clf1, clf2, clf3, clf4], labels, itertools.product([0, 1], repeat=2)): clf.fit(X, y) ax = plt.subplot(gs[grd[0], grd[1]]) fig = plot_decision_regions(X=X, y=y, clf=clf, legend=2) plt.title(lab) plt.show()","title":"Concentric Circles"},{"location":"user_guide/plotting/plot_decision_regions/#example-6-working-with-existing-axes-objects-using-subplots","text":"import matplotlib.pyplot as plt from mlxtend.plotting import plot_decision_regions from sklearn.linear_model import LogisticRegression from sklearn.naive_bayes import GaussianNB from sklearn import datasets import numpy as np # Loading some example data iris = datasets.load_iris() X = iris.data[:, 2] X = X[:, None] y = iris.target # Initializing and fitting classifiers clf1 = LogisticRegression(random_state=1, solver='lbfgs', multi_class='multinomial') clf2 = GaussianNB() clf1.fit(X, y) clf2.fit(X, y) fig, axes = plt.subplots(1, 2, figsize=(10, 3)) fig = plot_decision_regions(X=X, y=y, clf=clf1, ax=axes[0], legend=2) fig = plot_decision_regions(X=X, y=y, clf=clf2, ax=axes[1], legend=1) plt.show()","title":"Example 6 - Working with existing axes objects using subplots"},{"location":"user_guide/plotting/plot_decision_regions/#example-7-decision-regions-with-more-than-two-training-features","text":"from mlxtend.plotting import plot_decision_regions import matplotlib.pyplot as plt from sklearn import datasets from sklearn.svm import SVC # Loading some example data X, y = datasets.make_blobs(n_samples=600, n_features=3, centers=[[2, 2, -2],[-2, -2, 2]], cluster_std=[2, 2], random_state=2) # Training a classifier svm = SVC(gamma='auto') svm.fit(X, y) # Plotting decision regions fig, ax = plt.subplots() # Decision region for feature 3 = 1.5 value = 1.5 # Plot training sample with feature 3 = 1.5 +/- 0.75 width = 0.75 plot_decision_regions(X, y, clf=svm, filler_feature_values={2: value}, filler_feature_ranges={2: width}, legend=2, ax=ax) ax.set_xlabel('Feature 1') ax.set_ylabel('Feature 2') ax.set_title('Feature 3 = {}'.format(value)) # Adding axes annotations fig.suptitle('SVM on make_blobs') plt.show()","title":"Example 7 - Decision regions with more than two training features"},{"location":"user_guide/plotting/plot_decision_regions/#example-8-grid-of-decision-region-slices","text":"from mlxtend.plotting import plot_decision_regions import matplotlib.pyplot as plt from sklearn import datasets from sklearn.svm import SVC # Loading some example data X, y = datasets.make_blobs(n_samples=500, n_features=3, centers=[[2, 2, -2],[-2, -2, 2]], cluster_std=[2, 2], random_state=2) # Training a classifier svm = SVC(gamma='auto') svm.fit(X, y) # Plotting decision regions fig, axarr = plt.subplots(2, 2, figsize=(10,8), sharex=True, sharey=True) values = [-4.0, -1.0, 1.0, 4.0] width = 0.75 for value, ax in zip(values, axarr.flat): plot_decision_regions(X, y, clf=svm, filler_feature_values={2: value}, filler_feature_ranges={2: width}, legend=2, ax=ax) ax.set_xlabel('Feature 1') ax.set_ylabel('Feature 2') ax.set_title('Feature 3 = {}'.format(value)) # Adding axes annotations fig.suptitle('SVM on make_blobs') plt.show()","title":"Example 8 - Grid of decision region slices"},{"location":"user_guide/plotting/plot_decision_regions/#example-9-customizing-the-plotting-style","text":"from mlxtend.plotting import plot_decision_regions from mlxtend.preprocessing import shuffle_arrays_unison import matplotlib.pyplot as plt from sklearn import datasets from sklearn.svm import SVC # Loading some example data iris = datasets.load_iris() X = iris.data[:, [0, 2]] y = iris.target X, y = shuffle_arrays_unison(arrays=[X, y], random_seed=3) X_train, y_train = X[:100], y[:100] X_test, y_test = X[100:], y[100:] # Training a classifier svm = SVC(C=0.5, kernel='linear') svm.fit(X_train, y_train) # Specify keyword arguments to be passed to underlying plotting functions scatter_kwargs = {'s': 120, 'edgecolor': None, 'alpha': 0.7} contourf_kwargs = {'alpha': 0.2} scatter_highlight_kwargs = {'s': 120, 'label': 'Test data', 'alpha': 0.7} # Plotting decision regions plot_decision_regions(X, y, clf=svm, legend=2, X_highlight=X_test, scatter_kwargs=scatter_kwargs, contourf_kwargs=contourf_kwargs, scatter_highlight_kwargs=scatter_highlight_kwargs) # Adding axes annotations plt.xlabel('sepal length [cm]') plt.ylabel('petal length [cm]') plt.title('SVM on Iris') plt.show()","title":"Example 9 - Customizing the plotting style"},{"location":"user_guide/plotting/plot_decision_regions/#example-10-providing-your-own-legend-labels","text":"Custom legend labels can be provided by returning the axis object(s) from the plot_decision_region function and then getting the handles and labels of the legend. Custom handles (i.e., labels) can then be provided via ax.legend ax = plot_decision_regions(X, y, clf=svm, legend=0) handles, labels = ax.get_legend_handles_labels() ax.legend(handles, ['class 0', 'class 1', 'class 2'], framealpha=0.3, scatterpoints=1) An example is shown below. from mlxtend.plotting import plot_decision_regions import matplotlib.pyplot as plt from sklearn import datasets from sklearn.svm import SVC # Loading some example data iris = datasets.load_iris() X = iris.data[:, [0, 2]] y = iris.target # Training a classifier svm = SVC(C=0.5, kernel='linear') svm.fit(X, y) # Plotting decision regions ax = plot_decision_regions(X, y, clf=svm, legend=0) # Adding axes annotations plt.xlabel('sepal length [cm]') plt.ylabel('petal length [cm]') plt.title('SVM on Iris') handles, labels = ax.get_legend_handles_labels() ax.legend(handles, ['class square', 'class triangle', 'class circle'], framealpha=0.3, scatterpoints=1) plt.show()","title":"Example 10 - Providing your own legend labels"},{"location":"user_guide/plotting/plot_decision_regions/#example-11-plots-with-different-zoom-factors","text":"from mlxtend.plotting import plot_decision_regions import matplotlib.pyplot as plt from sklearn import datasets from sklearn.svm import SVC # Loading some example data iris = datasets.load_iris() X = iris.data[:, [0, 2]] y = iris.target # Training a classifier svm = SVC(C=0.5, kernel='linear') svm.fit(X, y) #sk-container-id-1 {color: black;background-color: white;}#sk-container-id-1 pre{padding: 0;}#sk-container-id-1 div.sk-toggleable {background-color: white;}#sk-container-id-1 label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.3em;box-sizing: border-box;text-align: center;}#sk-container-id-1 label.sk-toggleable__label-arrow:before {content: \"\u25b8\";float: left;margin-right: 0.25em;color: #696969;}#sk-container-id-1 label.sk-toggleable__label-arrow:hover:before {color: black;}#sk-container-id-1 div.sk-estimator:hover label.sk-toggleable__label-arrow:before {color: black;}#sk-container-id-1 div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}#sk-container-id-1 div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}#sk-container-id-1 input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}#sk-container-id-1 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {content: \"\u25be\";}#sk-container-id-1 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}#sk-container-id-1 div.sk-estimator {font-family: monospace;background-color: #f0f8ff;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;margin-bottom: 0.5em;}#sk-container-id-1 div.sk-estimator:hover {background-color: #d4ebff;}#sk-container-id-1 div.sk-parallel-item::after {content: \"\";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}#sk-container-id-1 div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 div.sk-serial::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: 0;}#sk-container-id-1 div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;padding-right: 0.2em;padding-left: 0.2em;position: relative;}#sk-container-id-1 div.sk-item {position: relative;z-index: 1;}#sk-container-id-1 div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;position: relative;}#sk-container-id-1 div.sk-item::before, #sk-container-id-1 div.sk-parallel-item::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: -1;}#sk-container-id-1 div.sk-parallel-item {display: flex;flex-direction: column;z-index: 1;position: relative;background-color: white;}#sk-container-id-1 div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}#sk-container-id-1 div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}#sk-container-id-1 div.sk-parallel-item:only-child::after {width: 0;}#sk-container-id-1 div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0 0.4em 0.5em 0.4em;box-sizing: border-box;padding-bottom: 0.4em;background-color: white;}#sk-container-id-1 div.sk-label label {font-family: monospace;font-weight: bold;display: inline-block;line-height: 1.2em;}#sk-container-id-1 div.sk-label-container {text-align: center;}#sk-container-id-1 div.sk-container {/* jupyter's `normalize.less` sets `[hidden] { display: none; }` but bootstrap.min.css set `[hidden] { display: none !important; }` so we also need the `!important` here to be able to override the default hidden behavior on the sphinx rendered scikit-learn.org. See: https://github.com/scikit-learn/scikit-learn/issues/21755 */display: inline-block !important;position: relative;}#sk-container-id-1 div.sk-text-repr-fallback {display: none;} SVC(C=0.5, kernel='linear') In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org. SVC SVC(C=0.5, kernel='linear') Default Zoom Factor plot_decision_regions(X, y, clf=svm, zoom_factor=1.) plt.show() Zooming out plot_decision_regions(X, y, clf=svm, zoom_factor=0.1) plt.show() Zooming in Note that while zooming in (by choosing a zoom_factor > 1.0) the plots are still created such that all data points are shown in the plot. plot_decision_regions(X, y, clf=svm, zoom_factor=2.0) plt.show() Cropping the axes In order to zoom in further, which means that some training examples won't be shown, you can simply crop the axes as shown below: plot_decision_regions(X, y, clf=svm, zoom_factor=2.0) plt.xlim(5, 6) plt.ylim(2, 5) plt.show()","title":"Example 11 - Plots with different zoom factors"},{"location":"user_guide/plotting/plot_decision_regions/#example-12-using-classifiers-that-expect-onehot-encoded-outputs-keras","text":"Most objects for classification that mimick the scikit-learn estimator API should be compatible with the plot_decision_regions function. However, if the classification model (e.g., a typical Keras model) output onehot-encoded predictions, we have to use an additional trick. I.e., for onehot encoded outputs, we need to wrap the Keras model into a class that converts these onehot encoded variables into integers. Such a wrapper class can be as simple as the following: class Onehot2Int(object): def __init__(self, model): self.model = model def predict(self, X): y_pred = self.model.predict(X) return np.argmax(y_pred, axis=1) The example below illustrates how the Onehot2Int class can be used with a Keras model that outputs onehot encoded labels: import keras from keras.models import Sequential from keras.layers import Dense import matplotlib.pyplot as plt import numpy as np from mlxtend.data import iris_data from mlxtend.preprocessing import standardize from mlxtend.plotting import plot_decision_regions from keras.utils import to_categorical X, y = iris_data() X = X[:, [2, 3]] X = standardize(X) # OneHot encoding y_onehot = to_categorical(y) # Create the model np.random.seed(123) model = Sequential() model.add(Dense(8, input_shape=(2,), activation='relu', kernel_initializer='he_uniform')) model.add(Dense(4, activation='relu', kernel_initializer='he_uniform')) model.add(Dense(3, activation='softmax')) # Configure the model and start training model.compile(loss=\"categorical_crossentropy\", optimizer=keras.optimizers.Adam(lr=0.005), metrics=['accuracy']) history = model.fit(X, y_onehot, epochs=10, batch_size=5, verbose=1, validation_split=0.1) Epoch 1/10 1/27 [>.............................] - ETA: 3s - loss: 1.2769 - accuracy: 0.4000 /Users/sebastianraschka/miniforge3/envs/mlxtend/lib/python3.8/site-packages/keras/optimizers/optimizer_v2/adam.py:117: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead. super().__init__(name, **kwargs) 2023-03-28 17:48:13.901264: W tensorflow/tsl/platform/profile_utils/cpu_utils.cc:128] Failed to get CPU frequency: 0 Hz 27/27 [==============================] - 0s 3ms/step - loss: 0.9526 - accuracy: 0.4222 - val_loss: 1.2656 - val_accuracy: 0.0000e+00 Epoch 2/10 27/27 [==============================] - 0s 834us/step - loss: 0.7062 - accuracy: 0.6741 - val_loss: 1.0939 - val_accuracy: 0.0000e+00 Epoch 3/10 27/27 [==============================] - 0s 808us/step - loss: 0.6461 - accuracy: 0.7111 - val_loss: 1.0705 - val_accuracy: 0.0667 Epoch 4/10 27/27 [==============================] - 0s 767us/step - loss: 0.6145 - accuracy: 0.7185 - val_loss: 1.0518 - val_accuracy: 0.0000e+00 Epoch 5/10 27/27 [==============================] - 0s 746us/step - loss: 0.5877 - accuracy: 0.7185 - val_loss: 1.0470 - val_accuracy: 0.0000e+00 Epoch 6/10 27/27 [==============================] - 0s 740us/step - loss: 0.5496 - accuracy: 0.7333 - val_loss: 1.0275 - val_accuracy: 0.0000e+00 Epoch 7/10 27/27 [==============================] - 0s 734us/step - loss: 0.4985 - accuracy: 0.7333 - val_loss: 1.0131 - val_accuracy: 0.0000e+00 Epoch 8/10 27/27 [==============================] - 0s 739us/step - loss: 0.4365 - accuracy: 0.7333 - val_loss: 0.9634 - val_accuracy: 0.0000e+00 Epoch 9/10 27/27 [==============================] - 0s 729us/step - loss: 0.3875 - accuracy: 0.7333 - val_loss: 0.9442 - val_accuracy: 0.0000e+00 Epoch 10/10 27/27 [==============================] - 0s 764us/step - loss: 0.3402 - accuracy: 0.7407 - val_loss: 0.8565 - val_accuracy: 0.0000e+00 # Wrap keras model model_no_ohe = Onehot2Int(model) # Plot decision boundary plot_decision_regions(X, y, clf=model_no_ohe) plt.show() 9600/9600 [==============================] - 3s 289us/step","title":"Example 12 - Using classifiers that expect onehot-encoded outputs (Keras)"},{"location":"user_guide/plotting/plot_decision_regions/#api","text":"plot_decision_regions(X, y, clf, feature_index=None, filler_feature_values=None, filler_feature_ranges=None, ax=None, X_highlight=None, zoom_factor=1.0, legend=1, hide_spines=True, markers='s^oxv<>', colors='#1f77b4,#ff7f0e,#3ca02c,#d62728,#9467bd,#8c564b,#e377c2,#7f7f7f,#bcbd22,#17becf', scatter_kwargs=None, contourf_kwargs=None, contour_kwargs=None, scatter_highlight_kwargs=None, n_jobs=None) Plot decision regions of a classifier. Please note that this functions assumes that class labels are labeled consecutively, e.g,. 0, 1, 2, 3, 4, and 5. If you have class labels with integer labels > 4, you may want to provide additional colors and/or markers as `colors` and `markers` arguments. See https://matplotlib.org/examples/color/named_colors.html for more information. Parameters X : array-like, shape = [n_samples, n_features] Feature Matrix. y : array-like, shape = [n_samples] True class labels. clf : Classifier object. Must have a .predict method. feature_index : array-like (default: (0,) for 1D, (0, 1) otherwise) Feature indices to use for plotting. The first index in feature_index will be on the x-axis, the second index will be on the y-axis. filler_feature_values : dict (default: None) Only needed for number features > 2. Dictionary of feature index-value pairs for the features not being plotted. filler_feature_ranges : dict (default: None) Only needed for number features > 2. Dictionary of feature index-value pairs for the features not being plotted. Will use the ranges provided to select training samples for plotting. ax : matplotlib.axes.Axes (default: None) An existing matplotlib Axes. Creates one if ax=None. X_highlight : array-like, shape = [n_samples, n_features] (default: None) An array with data points that are used to highlight samples in X . zoom_factor : float (default: 1.0) Controls the scale of the x- and y-axis of the decision plot. hide_spines : bool (default: True) Hide axis spines if True. legend : int (default: 1) Integer to specify the legend location. No legend if legend is 0. markers : str (default: 's^oxv<>') Scatterplot markers. colors : str (default: 'red,blue,limegreen,gray,cyan') Comma separated list of colors. scatter_kwargs : dict (default: None) Keyword arguments for underlying matplotlib scatter function. contourf_kwargs : dict (default: None) Keyword arguments for underlying matplotlib contourf function. contour_kwargs : dict (default: None) Keyword arguments for underlying matplotlib contour function (which draws the lines between decision regions). scatter_highlight_kwargs : dict (default: None) Keyword arguments for underlying matplotlib scatter function. n_jobs : int or None, optional (default=None) The number of CPUs to use to do the computation using Python's multiprocessing library. None means 1. -1 means using all processors. New in v0.22.0. Returns ax : matplotlib.axes.Axes object Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/plotting/plot_decision_regions/","title":"API"},{"location":"user_guide/plotting/plot_learning_curves/","text":"plot_learning_curves: Plot learning curves from training and test sets A function to plot learning curves for classifiers. Learning curves are extremely useful to analyze if a model is suffering from over- or under-fitting (high variance or high bias). The function can be imported via from mlxtend.plotting import plot_learning_curves This function uses the traditional holdout method based on a training and a test (or validation) set. The test set is kept constant while the size of the training set is increased gradually. The model is fit on the training set (of varying size) and evaluated on the same test set. The learning curve can be used as follows to diagnose overfitting: If there is a large gap between the training and test performance, then the model is likely suffering from overfitting. If both the training and test error are very large, the model is likely underfitting the data. The learning curve can also be used to judge whether collecting more data can be useful. More about that in Example 1 below. References - Example 1 The following code illustrates how we can construct a learning curve for a 5000-sample subset of the MNIST dataset. 4000 examples are used for training, and 1000 examples are reserved for testing. from mlxtend.plotting import plot_learning_curves import matplotlib.pyplot as plt from mlxtend.data import mnist_data from mlxtend.preprocessing import shuffle_arrays_unison from sklearn.neighbors import KNeighborsClassifier import numpy as np # Loading some example data X, y = mnist_data() X, y = shuffle_arrays_unison(arrays=[X, y], random_seed=123) X_train, X_test = X[:4000], X[4000:] y_train, y_test = y[:4000], y[4000:] clf = KNeighborsClassifier(n_neighbors=7) plot_learning_curves(X_train, y_train, X_test, y_test, clf) plt.show() As we can see from the plot above, the KNN model could benefit from additional training data. I.e., the slope indicates that if we had a larger training set, the test set error might decrease further. Also, based on the gap between training and test set performance, the model overfits slightly. This could pontentially be addressed by increasing the number of neighbors ( n_neighbors ) in KNN. While it is not relevant for analyzing the performance of the classifier, the region ~20% training set size shows that the model is underfitting (large training and test error), which is probably due to the small dataset size. API plot_learning_curves(X_train, y_train, X_test, y_test, clf, train_marker='o', test_marker='^', scoring='misclassification error', suppress_plot=False, print_model=True, title_fontsize=12, style='default', legend_loc='best') Plots learning curves of a classifier. Parameters X_train : array-like, shape = [n_samples, n_features] Feature matrix of the training dataset. y_train : array-like, shape = [n_samples] True class labels of the training dataset. X_test : array-like, shape = [n_samples, n_features] Feature matrix of the test dataset. y_test : array-like, shape = [n_samples] True class labels of the test dataset. clf : Classifier object. Must have a .predict .fit method. train_marker : str (default: 'o') Marker for the training set line plot. test_marker : str (default: '^') Marker for the test set line plot. scoring : str (default: 'misclassification error') If not 'misclassification error', accepts the following metrics (from scikit-learn): {'accuracy', 'average_precision', 'f1_micro', 'f1_macro', 'f1_weighted', 'f1_samples', 'log_loss', 'precision', 'recall', 'roc_auc', 'adjusted_rand_score', 'mean_absolute_error', 'mean_squared_error', 'median_absolute_error', 'r2'} suppress_plot=False : bool (default: False) Suppress matplotlib plots if True. Recommended for testing purposes. print_model : bool (default: True) Print model parameters in plot title if True. style : str (default: 'default') Matplotlib style. For more styles, please see https://matplotlib.org/stable/gallery/style_sheets/style_sheets_reference.html legend_loc : str (default: 'best') Where to place the plot legend: {'best', 'upper left', 'upper right', 'lower left', 'lower right'} Returns errors : (training_error, test_error): tuple of lists Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/plotting/plot_learning_curves/","title":"plot_learning_curves: Plot learning curves from training and test sets"},{"location":"user_guide/plotting/plot_learning_curves/#plot_learning_curves-plot-learning-curves-from-training-and-test-sets","text":"A function to plot learning curves for classifiers. Learning curves are extremely useful to analyze if a model is suffering from over- or under-fitting (high variance or high bias). The function can be imported via from mlxtend.plotting import plot_learning_curves This function uses the traditional holdout method based on a training and a test (or validation) set. The test set is kept constant while the size of the training set is increased gradually. The model is fit on the training set (of varying size) and evaluated on the same test set. The learning curve can be used as follows to diagnose overfitting: If there is a large gap between the training and test performance, then the model is likely suffering from overfitting. If both the training and test error are very large, the model is likely underfitting the data. The learning curve can also be used to judge whether collecting more data can be useful. More about that in Example 1 below.","title":"plot_learning_curves: Plot learning curves from training and test sets"},{"location":"user_guide/plotting/plot_learning_curves/#references","text":"-","title":"References"},{"location":"user_guide/plotting/plot_learning_curves/#example-1","text":"The following code illustrates how we can construct a learning curve for a 5000-sample subset of the MNIST dataset. 4000 examples are used for training, and 1000 examples are reserved for testing. from mlxtend.plotting import plot_learning_curves import matplotlib.pyplot as plt from mlxtend.data import mnist_data from mlxtend.preprocessing import shuffle_arrays_unison from sklearn.neighbors import KNeighborsClassifier import numpy as np # Loading some example data X, y = mnist_data() X, y = shuffle_arrays_unison(arrays=[X, y], random_seed=123) X_train, X_test = X[:4000], X[4000:] y_train, y_test = y[:4000], y[4000:] clf = KNeighborsClassifier(n_neighbors=7) plot_learning_curves(X_train, y_train, X_test, y_test, clf) plt.show() As we can see from the plot above, the KNN model could benefit from additional training data. I.e., the slope indicates that if we had a larger training set, the test set error might decrease further. Also, based on the gap between training and test set performance, the model overfits slightly. This could pontentially be addressed by increasing the number of neighbors ( n_neighbors ) in KNN. While it is not relevant for analyzing the performance of the classifier, the region ~20% training set size shows that the model is underfitting (large training and test error), which is probably due to the small dataset size.","title":"Example 1"},{"location":"user_guide/plotting/plot_learning_curves/#api","text":"plot_learning_curves(X_train, y_train, X_test, y_test, clf, train_marker='o', test_marker='^', scoring='misclassification error', suppress_plot=False, print_model=True, title_fontsize=12, style='default', legend_loc='best') Plots learning curves of a classifier. Parameters X_train : array-like, shape = [n_samples, n_features] Feature matrix of the training dataset. y_train : array-like, shape = [n_samples] True class labels of the training dataset. X_test : array-like, shape = [n_samples, n_features] Feature matrix of the test dataset. y_test : array-like, shape = [n_samples] True class labels of the test dataset. clf : Classifier object. Must have a .predict .fit method. train_marker : str (default: 'o') Marker for the training set line plot. test_marker : str (default: '^') Marker for the test set line plot. scoring : str (default: 'misclassification error') If not 'misclassification error', accepts the following metrics (from scikit-learn): {'accuracy', 'average_precision', 'f1_micro', 'f1_macro', 'f1_weighted', 'f1_samples', 'log_loss', 'precision', 'recall', 'roc_auc', 'adjusted_rand_score', 'mean_absolute_error', 'mean_squared_error', 'median_absolute_error', 'r2'} suppress_plot=False : bool (default: False) Suppress matplotlib plots if True. Recommended for testing purposes. print_model : bool (default: True) Print model parameters in plot title if True. style : str (default: 'default') Matplotlib style. For more styles, please see https://matplotlib.org/stable/gallery/style_sheets/style_sheets_reference.html legend_loc : str (default: 'best') Where to place the plot legend: {'best', 'upper left', 'upper right', 'lower left', 'lower right'} Returns errors : (training_error, test_error): tuple of lists Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/plotting/plot_learning_curves/","title":"API"},{"location":"user_guide/plotting/plot_linear_regression/","text":"plot_linear_regression: A quick way for plotting linear regression fits A function to plot linear regression fits. from mlxtend.plotting import plot_linear_regression Overview The plot_linear_regression is a convenience function that uses scikit-learn's linear_model.LinearRegression to fit a linear model and SciPy's stats.pearsonr to calculate the correlation coefficient. References - Example 1 - Ordinary Least Squares Simple Linear Regression import matplotlib.pyplot as plt from mlxtend.plotting import plot_linear_regression import numpy as np X = np.array([4, 8, 13, 26, 31, 10, 8, 30, 18, 12, 20, 5, 28, 18, 6, 31, 12, 12, 27, 11, 6, 14, 25, 7, 13,4, 15, 21, 15]) y = np.array([14, 24, 22, 59, 66, 25, 18, 60, 39, 32, 53, 18, 55, 41, 28, 61, 35, 36, 52, 23, 19, 25, 73, 16, 32, 14, 31, 43, 34]) intercept, slope, corr_coeff = plot_linear_regression(X, y) plt.show() API plot_linear_regression(X, y, model=LinearRegression(copy_X=True, fit_intercept=True, n_jobs=1, normalize=False), corr_func='pearsonr', scattercolor='blue', fit_style='k--', legend=True, xlim='auto') Plot a linear regression line fit. Parameters X : numpy array, shape = [n_samples,] Samples. y : numpy array, shape (n_samples,) Target values model: object (default: sklearn.linear_model.LinearRegression) Estimator object for regression. Must implement a .fit() and .predict() method. corr_func: str or function (default: 'pearsonr') Uses pearsonr from scipy.stats if corr_func='pearsonr'. to compute the regression slope. If not 'pearsonr', the corr_func , the corr_func parameter expects a function of the form func( , ) as inputs, which is expected to return a tuple (<correlation_coefficient>, <some_unused_value>) . scattercolor: string (default: blue) Color of scatter plot points. fit_style: string (default: k--) Style for the line fit. legend: bool (default: True) Plots legend with corr_coeff coef., fit coef., and intercept values. xlim: array-like (x_min, x_max) or 'auto' (default: 'auto') X-axis limits for the linear line fit. Returns regression_fit : tuple intercept, slope, corr_coeff (float, float, float) Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/plotting/plot_linear_regression/","title":"plot_linear_regression: A quick way for plotting linear regression fits"},{"location":"user_guide/plotting/plot_linear_regression/#plot_linear_regression-a-quick-way-for-plotting-linear-regression-fits","text":"A function to plot linear regression fits. from mlxtend.plotting import plot_linear_regression","title":"plot_linear_regression: A quick way for plotting linear regression fits"},{"location":"user_guide/plotting/plot_linear_regression/#overview","text":"The plot_linear_regression is a convenience function that uses scikit-learn's linear_model.LinearRegression to fit a linear model and SciPy's stats.pearsonr to calculate the correlation coefficient.","title":"Overview"},{"location":"user_guide/plotting/plot_linear_regression/#references","text":"-","title":"References"},{"location":"user_guide/plotting/plot_linear_regression/#example-1-ordinary-least-squares-simple-linear-regression","text":"import matplotlib.pyplot as plt from mlxtend.plotting import plot_linear_regression import numpy as np X = np.array([4, 8, 13, 26, 31, 10, 8, 30, 18, 12, 20, 5, 28, 18, 6, 31, 12, 12, 27, 11, 6, 14, 25, 7, 13,4, 15, 21, 15]) y = np.array([14, 24, 22, 59, 66, 25, 18, 60, 39, 32, 53, 18, 55, 41, 28, 61, 35, 36, 52, 23, 19, 25, 73, 16, 32, 14, 31, 43, 34]) intercept, slope, corr_coeff = plot_linear_regression(X, y) plt.show()","title":"Example 1 - Ordinary Least Squares Simple Linear Regression"},{"location":"user_guide/plotting/plot_linear_regression/#api","text":"plot_linear_regression(X, y, model=LinearRegression(copy_X=True, fit_intercept=True, n_jobs=1, normalize=False), corr_func='pearsonr', scattercolor='blue', fit_style='k--', legend=True, xlim='auto') Plot a linear regression line fit. Parameters X : numpy array, shape = [n_samples,] Samples. y : numpy array, shape (n_samples,) Target values model: object (default: sklearn.linear_model.LinearRegression) Estimator object for regression. Must implement a .fit() and .predict() method. corr_func: str or function (default: 'pearsonr') Uses pearsonr from scipy.stats if corr_func='pearsonr'. to compute the regression slope. If not 'pearsonr', the corr_func , the corr_func parameter expects a function of the form func( , ) as inputs, which is expected to return a tuple (<correlation_coefficient>, <some_unused_value>) . scattercolor: string (default: blue) Color of scatter plot points. fit_style: string (default: k--) Style for the line fit. legend: bool (default: True) Plots legend with corr_coeff coef., fit coef., and intercept values. xlim: array-like (x_min, x_max) or 'auto' (default: 'auto') X-axis limits for the linear line fit. Returns regression_fit : tuple intercept, slope, corr_coeff (float, float, float) Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/plotting/plot_linear_regression/","title":"API"},{"location":"user_guide/plotting/plot_pca_correlation_graph/","text":"plot_pca_correlation_graph: plot correlations between original features and principal components A function to provide a correlation circle for PCA. > from mlxtend.plotting import plot_pca_correlation_graph In a so called correlation circle, the correlations between the original dataset features and the principal component(s) are shown via coordinates. Example The following correlation circle examples visualizes the correlation between the first two principal components and the 4 original iris dataset features. Features with a positive correlation will be grouped together. Totally uncorrelated features are orthogonal to each other. Features with a negative correlation will be plotted on the opposing quadrants of this plot. from mlxtend.data import iris_data from mlxtend.plotting import plot_pca_correlation_graph import numpy as np X, y = iris_data() X_norm = X / X.std(axis=0) # Normalizing the feature columns is recommended feature_names = [ 'sepal length', 'sepal width', 'petal length', 'petal width'] figure, correlation_matrix = plot_pca_correlation_graph(X_norm, feature_names, dimensions=(1, 2), figure_axis_size=10) correlation_matrix .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } Dim 1 Dim 2 sepal length -0.891224 -0.357352 sepal width 0.449313 -0.888351 petal length -0.991684 -0.020247 petal width -0.964996 -0.062786 Further, note that the percentage values shown on the x and y axis denote how much of the variance in the original dataset is explained by each principal component axis. I.e.., if PC1 lists 72.7% and PC2 lists 23.0% as shown above, then combined, the 2 principal components explain 95.7% of the total variance. API plot_pca_correlation_graph(X, variables_names, dimensions=(1, 2), figure_axis_size=6, X_pca=None, explained_variance=None) Compute the PCA for X and plots the Correlation graph Parameters X : 2d array like. The columns represent the different variables and the rows are the samples of thos variables variables_names : array like Name of the columns (the variables) of X dimensions: tuple with two elements. dimensions to be plotted (x,y) figure_axis_size : size of the final frame. The figure created is a square with length and width equal to figure_axis_size. X_pca : np.ndarray, shape = [n_samples, n_components]. Optional. X_pca is the matrix of the transformed components from X. If not provided, the function computes PCA automatically using mlxtend.feature_extraction.PrincipalComponentAnalysis Expected n_componentes >= max(dimensions) explained_variance : 1 dimension np.ndarray, length = n_components Optional. explained_variance are the eigenvalues from the diagonalized covariance matrix on the PCA transformatiopn. If not provided, the function computes PCA independently Expected n_componentes == X.shape[1] Returns matplotlib_figure, correlation_matrix Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/plotting/plot_pca_correlation_graph/","title":"plot_pca_correlation_graph: plot correlations between original features and principal components"},{"location":"user_guide/plotting/plot_pca_correlation_graph/#plot_pca_correlation_graph-plot-correlations-between-original-features-and-principal-components","text":"A function to provide a correlation circle for PCA. > from mlxtend.plotting import plot_pca_correlation_graph In a so called correlation circle, the correlations between the original dataset features and the principal component(s) are shown via coordinates.","title":"plot_pca_correlation_graph: plot correlations between original features and principal components"},{"location":"user_guide/plotting/plot_pca_correlation_graph/#example","text":"The following correlation circle examples visualizes the correlation between the first two principal components and the 4 original iris dataset features. Features with a positive correlation will be grouped together. Totally uncorrelated features are orthogonal to each other. Features with a negative correlation will be plotted on the opposing quadrants of this plot. from mlxtend.data import iris_data from mlxtend.plotting import plot_pca_correlation_graph import numpy as np X, y = iris_data() X_norm = X / X.std(axis=0) # Normalizing the feature columns is recommended feature_names = [ 'sepal length', 'sepal width', 'petal length', 'petal width'] figure, correlation_matrix = plot_pca_correlation_graph(X_norm, feature_names, dimensions=(1, 2), figure_axis_size=10) correlation_matrix .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } Dim 1 Dim 2 sepal length -0.891224 -0.357352 sepal width 0.449313 -0.888351 petal length -0.991684 -0.020247 petal width -0.964996 -0.062786 Further, note that the percentage values shown on the x and y axis denote how much of the variance in the original dataset is explained by each principal component axis. I.e.., if PC1 lists 72.7% and PC2 lists 23.0% as shown above, then combined, the 2 principal components explain 95.7% of the total variance.","title":"Example"},{"location":"user_guide/plotting/plot_pca_correlation_graph/#api","text":"plot_pca_correlation_graph(X, variables_names, dimensions=(1, 2), figure_axis_size=6, X_pca=None, explained_variance=None) Compute the PCA for X and plots the Correlation graph Parameters X : 2d array like. The columns represent the different variables and the rows are the samples of thos variables variables_names : array like Name of the columns (the variables) of X dimensions: tuple with two elements. dimensions to be plotted (x,y) figure_axis_size : size of the final frame. The figure created is a square with length and width equal to figure_axis_size. X_pca : np.ndarray, shape = [n_samples, n_components]. Optional. X_pca is the matrix of the transformed components from X. If not provided, the function computes PCA automatically using mlxtend.feature_extraction.PrincipalComponentAnalysis Expected n_componentes >= max(dimensions) explained_variance : 1 dimension np.ndarray, length = n_components Optional. explained_variance are the eigenvalues from the diagonalized covariance matrix on the PCA transformatiopn. If not provided, the function computes PCA independently Expected n_componentes == X.shape[1] Returns matplotlib_figure, correlation_matrix Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/plotting/plot_pca_correlation_graph/","title":"API"},{"location":"user_guide/plotting/plot_sequential_feature_selection/","text":"plot_sequential_feature_selection: Visualize selected feature subset performances from the SequentialFeatureSelector A matplotlib utility function for visualizing results from feature_selection.SequentialFeatureSelector . from mlxtend.plotting import plot_sequential_feature_selection Overview for more information on sequential feature selection, please see feature_selection.SequentialFeatureSelector . Example 1 - Plotting the results from SequentialFeatureSelector from mlxtend.plotting import plot_sequential_feature_selection as plot_sfs from mlxtend.feature_selection import SequentialFeatureSelector as SFS import matplotlib.pyplot as plt from sklearn.neighbors import KNeighborsClassifier from sklearn.datasets import load_iris iris = load_iris() X = iris.data y = iris.target knn = KNeighborsClassifier(n_neighbors=4) sfs = SFS(knn, k_features=4, forward=True, floating=False, scoring='accuracy', cv=5) sfs = sfs.fit(X, y) fig1 = plot_sfs(sfs.get_metric_dict(), kind='std_dev', figsize=(6, 4)) plt.ylim([0.8, 1]) plt.title('Sequential Forward Selection (w. StdDev)') plt.grid() plt.show() API plot_sequential_feature_selection(metric_dict, figsize=None, kind='std_dev', color='blue', bcolor='steelblue', marker='o', alpha=0.2, ylabel='Performance', confidence_interval=0.95) Plot feature selection results. Parameters metric_dict : mlxtend.SequentialFeatureSelector.get_metric_dict() object figsize : tuple (default: None) Height and width of the figure kind : str (default: \"std_dev\") The kind of error bar or confidence interval in {'std_dev', 'std_err', 'ci', None}. color : str (default: \"blue\") Color of the lineplot (accepts any matplotlib color name) bcolor : str (default: \"steelblue\"). Color of the error bars / confidence intervals (accepts any matplotlib color name). marker : str (default: \"o\") Marker of the line plot (accepts any matplotlib marker name). alpha : float in [0, 1] (default: 0.2) Transparency of the error bars / confidence intervals. ylabel : str (default: \"Performance\") Y-axis label. confidence_interval : float (default: 0.95) Confidence level if kind='ci' . Returns fig : matplotlib.pyplot.figure() object Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/plotting/plot_sequential_feature_selection/ ython","title":"plot_sequential_feature_selection: Visualize selected feature subset performances from the SequentialFeatureSelector"},{"location":"user_guide/plotting/plot_sequential_feature_selection/#plot_sequential_feature_selection-visualize-selected-feature-subset-performances-from-the-sequentialfeatureselector","text":"A matplotlib utility function for visualizing results from feature_selection.SequentialFeatureSelector . from mlxtend.plotting import plot_sequential_feature_selection","title":"plot_sequential_feature_selection: Visualize selected feature subset performances from the SequentialFeatureSelector"},{"location":"user_guide/plotting/plot_sequential_feature_selection/#overview","text":"for more information on sequential feature selection, please see feature_selection.SequentialFeatureSelector .","title":"Overview"},{"location":"user_guide/plotting/plot_sequential_feature_selection/#example-1-plotting-the-results-from-sequentialfeatureselector","text":"from mlxtend.plotting import plot_sequential_feature_selection as plot_sfs from mlxtend.feature_selection import SequentialFeatureSelector as SFS import matplotlib.pyplot as plt from sklearn.neighbors import KNeighborsClassifier from sklearn.datasets import load_iris iris = load_iris() X = iris.data y = iris.target knn = KNeighborsClassifier(n_neighbors=4) sfs = SFS(knn, k_features=4, forward=True, floating=False, scoring='accuracy', cv=5) sfs = sfs.fit(X, y) fig1 = plot_sfs(sfs.get_metric_dict(), kind='std_dev', figsize=(6, 4)) plt.ylim([0.8, 1]) plt.title('Sequential Forward Selection (w. StdDev)') plt.grid() plt.show()","title":"Example 1 - Plotting the results from SequentialFeatureSelector"},{"location":"user_guide/plotting/plot_sequential_feature_selection/#api","text":"plot_sequential_feature_selection(metric_dict, figsize=None, kind='std_dev', color='blue', bcolor='steelblue', marker='o', alpha=0.2, ylabel='Performance', confidence_interval=0.95) Plot feature selection results. Parameters metric_dict : mlxtend.SequentialFeatureSelector.get_metric_dict() object figsize : tuple (default: None) Height and width of the figure kind : str (default: \"std_dev\") The kind of error bar or confidence interval in {'std_dev', 'std_err', 'ci', None}. color : str (default: \"blue\") Color of the lineplot (accepts any matplotlib color name) bcolor : str (default: \"steelblue\"). Color of the error bars / confidence intervals (accepts any matplotlib color name). marker : str (default: \"o\") Marker of the line plot (accepts any matplotlib marker name). alpha : float in [0, 1] (default: 0.2) Transparency of the error bars / confidence intervals. ylabel : str (default: \"Performance\") Y-axis label. confidence_interval : float (default: 0.95) Confidence level if kind='ci' . Returns fig : matplotlib.pyplot.figure() object Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/plotting/plot_sequential_feature_selection/ ython","title":"API"},{"location":"user_guide/plotting/scatter_hist/","text":"scatter_hist: create a scatter histogram plot A function to quickly produce a scatter histogram plot. from mlxtend.plotting import scatter_hist Overview References https://matplotlib.org/gallery/lines_bars_and_markers/scatter_hist.html Example 1 - Scatter Plot and Histograms from Pandas DataFrames from mlxtend.data import iris_data from mlxtend.plotting import scatter_hist import pandas as pd X, y = iris_data() df = pd.DataFrame(X) df.columns = ['sepal length [cm]', 'sepal width [cm]', 'petal length [cm]', 'petal width [cm]'] df.head(5) .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } sepal length [cm] sepal width [cm] petal length [cm] petal width [cm] 0 5.1 3.5 1.4 0.2 1 4.9 3.0 1.4 0.2 2 4.7 3.2 1.3 0.2 3 4.6 3.1 1.5 0.2 4 5.0 3.6 1.4 0.2 import matplotlib.pyplot as plt from mlxtend.plotting import scatter_hist fig = scatter_hist(df[\"sepal length [cm]\"], df[\"sepal width [cm]\"]) Example 2 - Category Scatter from NumPy Arrays from mlxtend.data import iris_data from mlxtend.plotting import scatter_hist import pandas as pd X, y = iris_data() X[:5] array([[5.1, 3.5, 1.4, 0.2], [4.9, 3. , 1.4, 0.2], [4.7, 3.2, 1.3, 0.2], [4.6, 3.1, 1.5, 0.2], [5. , 3.6, 1.4, 0.2]]) fig = scatter_hist(X[:, 0], X[:, 1]) API scatter_hist(x, y, xlabel=None, ylabel=None, figsize=(5, 5)) Scatter plot and individual feature histograms along axes. Parameters x : 1D array-like or Pandas Series X-axis values. y : 1D array-like or Pandas Series Y-axis values. xlabel : str (default: None ) Label for the X-axis values. If x is a pandas Series, and xlabel is None , the label is inferred automatically. ylabel : str (default: None ) Label for the X-axis values. If y is a pandas Series, and ylabel is None , the label is inferred automatically. figsize : tuple (default: (5, 5) ) Matplotlib figure size. Returns plot : Matplotlib Figure object","title":"scatter_hist: create a scatter histogram plot"},{"location":"user_guide/plotting/scatter_hist/#scatter_hist-create-a-scatter-histogram-plot","text":"A function to quickly produce a scatter histogram plot. from mlxtend.plotting import scatter_hist","title":"scatter_hist: create a scatter histogram plot"},{"location":"user_guide/plotting/scatter_hist/#overview","text":"","title":"Overview"},{"location":"user_guide/plotting/scatter_hist/#references","text":"https://matplotlib.org/gallery/lines_bars_and_markers/scatter_hist.html","title":"References"},{"location":"user_guide/plotting/scatter_hist/#example-1-scatter-plot-and-histograms-from-pandas-dataframes","text":"from mlxtend.data import iris_data from mlxtend.plotting import scatter_hist import pandas as pd X, y = iris_data() df = pd.DataFrame(X) df.columns = ['sepal length [cm]', 'sepal width [cm]', 'petal length [cm]', 'petal width [cm]'] df.head(5) .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } sepal length [cm] sepal width [cm] petal length [cm] petal width [cm] 0 5.1 3.5 1.4 0.2 1 4.9 3.0 1.4 0.2 2 4.7 3.2 1.3 0.2 3 4.6 3.1 1.5 0.2 4 5.0 3.6 1.4 0.2 import matplotlib.pyplot as plt from mlxtend.plotting import scatter_hist fig = scatter_hist(df[\"sepal length [cm]\"], df[\"sepal width [cm]\"])","title":"Example 1 - Scatter Plot and Histograms from Pandas DataFrames"},{"location":"user_guide/plotting/scatter_hist/#example-2-category-scatter-from-numpy-arrays","text":"from mlxtend.data import iris_data from mlxtend.plotting import scatter_hist import pandas as pd X, y = iris_data() X[:5] array([[5.1, 3.5, 1.4, 0.2], [4.9, 3. , 1.4, 0.2], [4.7, 3.2, 1.3, 0.2], [4.6, 3.1, 1.5, 0.2], [5. , 3.6, 1.4, 0.2]]) fig = scatter_hist(X[:, 0], X[:, 1])","title":"Example 2 - Category Scatter from NumPy Arrays"},{"location":"user_guide/plotting/scatter_hist/#api","text":"scatter_hist(x, y, xlabel=None, ylabel=None, figsize=(5, 5)) Scatter plot and individual feature histograms along axes. Parameters x : 1D array-like or Pandas Series X-axis values. y : 1D array-like or Pandas Series Y-axis values. xlabel : str (default: None ) Label for the X-axis values. If x is a pandas Series, and xlabel is None , the label is inferred automatically. ylabel : str (default: None ) Label for the X-axis values. If y is a pandas Series, and ylabel is None , the label is inferred automatically. figsize : tuple (default: (5, 5) ) Matplotlib figure size. Returns plot : Matplotlib Figure object","title":"API"},{"location":"user_guide/plotting/scatterplotmatrix/","text":"scatterplotmatrix: visualize datasets via a scatter plot matrix A function for creating as scatter plot matrix in matplotlib from mlxtend.plotting import scatterplotmatrix Overview A matplotlib convenience function for creating a scatterplot matrix. References - Example 1 - Simple Scatter Plot Matrix import matplotlib.pyplot as plt from mlxtend.data import iris_data from mlxtend.plotting import scatterplotmatrix X, y = iris_data() scatterplotmatrix(X, figsize=(10, 8)) plt.tight_layout() plt.show() Example 2 - Scatter Plot Matrix with Multiple Categories names = ['sepal length [cm]', 'sepal width [cm]', 'petal length [cm]', 'petal width [cm]'] fig, axes = scatterplotmatrix(X[y==0], figsize=(10, 8), alpha=0.5) fig, axes = scatterplotmatrix(X[y==1], fig_axes=(fig, axes), alpha=0.5) fig, axes = scatterplotmatrix(X[y==2], fig_axes=(fig, axes), alpha=0.5, names=names) plt.tight_layout() plt.show() API scatterplotmatrix(X, fig_axes=None, names=None, figsize=(8, 8), alpha=1.0, kwargs) Lower triangular of a scatterplot matrix Parameters X : array-like, shape={num_examples, num_features} Design matrix containing data instances (examples) with multiple exploratory variables (features). fix_axes : tuple (default: None) A (fig, axes) tuple, where fig is an figure object and axes is an axes object created via matplotlib, for example, by calling the pyplot subplot function fig, axes = plt.subplots(...) names : list (default: None) A list of string names, which should have the same number of elements as there are features (columns) in X . figsize : tuple (default: (8, 8)) Height and width of the subplot grid. Ignored if fig_axes is not None . alpha : float (default: 1.0) Transparency for both the scatter plots and the histograms along the diagonal. **kwargs : kwargs Keyword arguments for the scatterplots. Returns fix_axes : tuple A (fig, axes) tuple, where fig is an figure object and axes is an axes object created via matplotlib, for example, by calling the pyplot subplot function fig, axes = plt.subplots(...)","title":"scatterplotmatrix: visualize datasets via a scatter plot matrix"},{"location":"user_guide/plotting/scatterplotmatrix/#scatterplotmatrix-visualize-datasets-via-a-scatter-plot-matrix","text":"A function for creating as scatter plot matrix in matplotlib from mlxtend.plotting import scatterplotmatrix","title":"scatterplotmatrix: visualize datasets via a scatter plot matrix"},{"location":"user_guide/plotting/scatterplotmatrix/#overview","text":"A matplotlib convenience function for creating a scatterplot matrix.","title":"Overview"},{"location":"user_guide/plotting/scatterplotmatrix/#references","text":"-","title":"References"},{"location":"user_guide/plotting/scatterplotmatrix/#example-1-simple-scatter-plot-matrix","text":"import matplotlib.pyplot as plt from mlxtend.data import iris_data from mlxtend.plotting import scatterplotmatrix X, y = iris_data() scatterplotmatrix(X, figsize=(10, 8)) plt.tight_layout() plt.show()","title":"Example 1 - Simple Scatter Plot Matrix"},{"location":"user_guide/plotting/scatterplotmatrix/#example-2-scatter-plot-matrix-with-multiple-categories","text":"names = ['sepal length [cm]', 'sepal width [cm]', 'petal length [cm]', 'petal width [cm]'] fig, axes = scatterplotmatrix(X[y==0], figsize=(10, 8), alpha=0.5) fig, axes = scatterplotmatrix(X[y==1], fig_axes=(fig, axes), alpha=0.5) fig, axes = scatterplotmatrix(X[y==2], fig_axes=(fig, axes), alpha=0.5, names=names) plt.tight_layout() plt.show()","title":"Example 2 - Scatter Plot Matrix with Multiple Categories"},{"location":"user_guide/plotting/scatterplotmatrix/#api","text":"scatterplotmatrix(X, fig_axes=None, names=None, figsize=(8, 8), alpha=1.0, kwargs) Lower triangular of a scatterplot matrix Parameters X : array-like, shape={num_examples, num_features} Design matrix containing data instances (examples) with multiple exploratory variables (features). fix_axes : tuple (default: None) A (fig, axes) tuple, where fig is an figure object and axes is an axes object created via matplotlib, for example, by calling the pyplot subplot function fig, axes = plt.subplots(...) names : list (default: None) A list of string names, which should have the same number of elements as there are features (columns) in X . figsize : tuple (default: (8, 8)) Height and width of the subplot grid. Ignored if fig_axes is not None . alpha : float (default: 1.0) Transparency for both the scatter plots and the histograms along the diagonal. **kwargs : kwargs Keyword arguments for the scatterplots. Returns fix_axes : tuple A (fig, axes) tuple, where fig is an figure object and axes is an axes object created via matplotlib, for example, by calling the pyplot subplot function fig, axes = plt.subplots(...)","title":"API"},{"location":"user_guide/plotting/stacked_barplot/","text":"stacked_barplot: Plot stacked bar plots in matplotlib A function to conveniently plot stacked bar plots in matplotlib using pandas DataFrame s. from mlxtend.plotting import stacked_barplot Overview A matplotlib convenience function for creating barplots from DataFrames where each sample is associated with several categories. References - Example 1 - Stacked Barplot from Pandas DataFrames import pandas as pd s1 = [1.0, 2.0, 3.0, 4.0] s2 = [1.4, 2.1, 2.9, 5.1] s3 = [1.9, 2.2, 3.5, 4.1] s4 = [1.4, 2.5, 3.5, 4.2] data = [s1, s2, s3, s4] df = pd.DataFrame(data, columns=['X1', 'X2', 'X3', 'X4']) df.columns = ['X1', 'X2', 'X3', 'X4'] df.index = ['Sample1', 'Sample2', 'Sample3', 'Sample4'] df X1 X2 X3 X4 Sample1 1.0 2.0 3.0 4.0 Sample2 1.4 2.1 2.9 5.1 Sample3 1.9 2.2 3.5 4.1 Sample4 1.4 2.5 3.5 4.2 By default, the index of the DataFrame is used as column labels, and the DataFrame columns are used for the plot legend. import matplotlib.pyplot as plt from mlxtend.plotting import stacked_barplot fig = stacked_barplot(df, rotation=45, legend_loc='best') API stacked_barplot(df, bar_width='auto', colors='bgrcky', labels='index', rotation=90, legend_loc='best') Function to plot stacked barplots Parameters df : pandas.DataFrame A pandas DataFrame where the index denotes the x-axis labels, and the columns contain the different measurements for each row. bar_width: 'auto' or float (default: 'auto') Parameter to set the widths of the bars. if 'auto', the width is automatically determined by the number of columns in the dataset. colors: str (default: 'bgrcky') The colors of the bars. labels: 'index' or iterable (default: 'index') If 'index', the DataFrame index will be used as x-tick labels. rotation: int (default: 90) Parameter to rotate the x-axis labels. legend_loc : str (default: 'best') Location of the plot legend {best, upper left, upper right, lower left, lower right} No legend if legend_loc=False Returns fig : matplotlib.pyplot figure object Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/plotting/stacked_barplot/","title":"stacked_barplot: Plot stacked bar plots in matplotlib"},{"location":"user_guide/plotting/stacked_barplot/#stacked_barplot-plot-stacked-bar-plots-in-matplotlib","text":"A function to conveniently plot stacked bar plots in matplotlib using pandas DataFrame s. from mlxtend.plotting import stacked_barplot","title":"stacked_barplot: Plot stacked bar plots in matplotlib"},{"location":"user_guide/plotting/stacked_barplot/#overview","text":"A matplotlib convenience function for creating barplots from DataFrames where each sample is associated with several categories.","title":"Overview"},{"location":"user_guide/plotting/stacked_barplot/#references","text":"-","title":"References"},{"location":"user_guide/plotting/stacked_barplot/#example-1-stacked-barplot-from-pandas-dataframes","text":"import pandas as pd s1 = [1.0, 2.0, 3.0, 4.0] s2 = [1.4, 2.1, 2.9, 5.1] s3 = [1.9, 2.2, 3.5, 4.1] s4 = [1.4, 2.5, 3.5, 4.2] data = [s1, s2, s3, s4] df = pd.DataFrame(data, columns=['X1', 'X2', 'X3', 'X4']) df.columns = ['X1', 'X2', 'X3', 'X4'] df.index = ['Sample1', 'Sample2', 'Sample3', 'Sample4'] df X1 X2 X3 X4 Sample1 1.0 2.0 3.0 4.0 Sample2 1.4 2.1 2.9 5.1 Sample3 1.9 2.2 3.5 4.1 Sample4 1.4 2.5 3.5 4.2 By default, the index of the DataFrame is used as column labels, and the DataFrame columns are used for the plot legend. import matplotlib.pyplot as plt from mlxtend.plotting import stacked_barplot fig = stacked_barplot(df, rotation=45, legend_loc='best')","title":"Example 1 - Stacked Barplot from Pandas DataFrames"},{"location":"user_guide/plotting/stacked_barplot/#api","text":"stacked_barplot(df, bar_width='auto', colors='bgrcky', labels='index', rotation=90, legend_loc='best') Function to plot stacked barplots Parameters df : pandas.DataFrame A pandas DataFrame where the index denotes the x-axis labels, and the columns contain the different measurements for each row. bar_width: 'auto' or float (default: 'auto') Parameter to set the widths of the bars. if 'auto', the width is automatically determined by the number of columns in the dataset. colors: str (default: 'bgrcky') The colors of the bars. labels: 'index' or iterable (default: 'index') If 'index', the DataFrame index will be used as x-tick labels. rotation: int (default: 90) Parameter to rotate the x-axis labels. legend_loc : str (default: 'best') Location of the plot legend {best, upper left, upper right, lower left, lower right} No legend if legend_loc=False Returns fig : matplotlib.pyplot figure object Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/plotting/stacked_barplot/","title":"API"},{"location":"user_guide/preprocessing/CopyTransformer/","text":"CopyTransformer: A function that creates a copy of the input array in a scikit-learn pipeline A simple transformer that returns a copy of the input array, for example, as part of a scikit-learn pipeline. from mlxtend.preprocessing import CopyTransformer Example 1 from sklearn.pipeline import Pipeline from sklearn.model_selection import GridSearchCV from sklearn.ensemble import RandomForestClassifier from sklearn.feature_extraction.text import CountVectorizer from mlxtend.preprocessing import CopyTransformer import re import numpy as np X_train = np.array(['abc def ghi', 'this is a test', 'this is a test', 'this is a test']) y_train = np.array([0, 0, 1, 1]) pipe_1 = Pipeline([ ('vect', CountVectorizer()), ('to_dense', CopyTransformer()), ('clf', RandomForestClassifier()) ]) parameters_1 = dict( clf__n_estimators=[50, 100, 200], clf__max_features=['sqrt', 'log2', None],) grid_search_1 = GridSearchCV(pipe_1, parameters_1, n_jobs=1, verbose=1, scoring='accuracy', cv=2) print(\"Performing grid search...\") print(\"pipeline:\", [name for name, _ in pipe_1.steps]) print(\"parameters:\") grid_search_1.fit(X_train, y_train) print(\"Best score: %0.3f\" % grid_search_1.best_score_) print(\"Best parameters set:\") best_parameters_1 = grid_search_1.best_estimator_.get_params() for param_name in sorted(parameters_1.keys()): print(\"\\t%s: %r\" % (param_name, best_parameters_1[param_name])) Performing grid search... pipeline: ['vect', 'to_dense', 'clf'] parameters: Fitting 2 folds for each of 9 candidates, totalling 18 fits Best score: 0.500 Best parameters set: clf__max_features: 'sqrt' clf__n_estimators: 50 [Parallel(n_jobs=1)]: Done 18 out of 18 | elapsed: 2.9s finished API CopyTransformer() Transformer that returns a copy of the input array For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/preprocessing/CopyTransformer/ Methods fit(X, y=None) Mock method. Does nothing. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] (default: None) Returns self fit_transform(X, y=None) Return a copy of the input array. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] (default: None) Returns X_copy : copy of the input X array. get_params(deep=True) Get parameters for this estimator. Parameters deep : boolean, optional If True, will return the parameters for this estimator and contained subobjects that are estimators. Returns params : mapping of string to any Parameter names mapped to their values. set_params( params) Set the parameters of this estimator. The method works on simple estimators as well as on nested objects (such as pipelines). The latter have parameters of the form <component>__<parameter> so that it's possible to update each component of a nested object. Returns self transform(X, y=None) Return a copy of the input array. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] (default: None) Returns X_copy : copy of the input X array.","title":"CopyTransformer: A function that creates a copy of the input array in a scikit-learn pipeline"},{"location":"user_guide/preprocessing/CopyTransformer/#copytransformer-a-function-that-creates-a-copy-of-the-input-array-in-a-scikit-learn-pipeline","text":"A simple transformer that returns a copy of the input array, for example, as part of a scikit-learn pipeline. from mlxtend.preprocessing import CopyTransformer","title":"CopyTransformer: A function that creates a copy of the input array in a scikit-learn pipeline"},{"location":"user_guide/preprocessing/CopyTransformer/#example-1","text":"from sklearn.pipeline import Pipeline from sklearn.model_selection import GridSearchCV from sklearn.ensemble import RandomForestClassifier from sklearn.feature_extraction.text import CountVectorizer from mlxtend.preprocessing import CopyTransformer import re import numpy as np X_train = np.array(['abc def ghi', 'this is a test', 'this is a test', 'this is a test']) y_train = np.array([0, 0, 1, 1]) pipe_1 = Pipeline([ ('vect', CountVectorizer()), ('to_dense', CopyTransformer()), ('clf', RandomForestClassifier()) ]) parameters_1 = dict( clf__n_estimators=[50, 100, 200], clf__max_features=['sqrt', 'log2', None],) grid_search_1 = GridSearchCV(pipe_1, parameters_1, n_jobs=1, verbose=1, scoring='accuracy', cv=2) print(\"Performing grid search...\") print(\"pipeline:\", [name for name, _ in pipe_1.steps]) print(\"parameters:\") grid_search_1.fit(X_train, y_train) print(\"Best score: %0.3f\" % grid_search_1.best_score_) print(\"Best parameters set:\") best_parameters_1 = grid_search_1.best_estimator_.get_params() for param_name in sorted(parameters_1.keys()): print(\"\\t%s: %r\" % (param_name, best_parameters_1[param_name])) Performing grid search... pipeline: ['vect', 'to_dense', 'clf'] parameters: Fitting 2 folds for each of 9 candidates, totalling 18 fits Best score: 0.500 Best parameters set: clf__max_features: 'sqrt' clf__n_estimators: 50 [Parallel(n_jobs=1)]: Done 18 out of 18 | elapsed: 2.9s finished","title":"Example 1"},{"location":"user_guide/preprocessing/CopyTransformer/#api","text":"CopyTransformer() Transformer that returns a copy of the input array For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/preprocessing/CopyTransformer/","title":"API"},{"location":"user_guide/preprocessing/CopyTransformer/#methods","text":"fit(X, y=None) Mock method. Does nothing. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] (default: None) Returns self fit_transform(X, y=None) Return a copy of the input array. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] (default: None) Returns X_copy : copy of the input X array. get_params(deep=True) Get parameters for this estimator. Parameters deep : boolean, optional If True, will return the parameters for this estimator and contained subobjects that are estimators. Returns params : mapping of string to any Parameter names mapped to their values. set_params( params) Set the parameters of this estimator. The method works on simple estimators as well as on nested objects (such as pipelines). The latter have parameters of the form <component>__<parameter> so that it's possible to update each component of a nested object. Returns self transform(X, y=None) Return a copy of the input array. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] (default: None) Returns X_copy : copy of the input X array.","title":"Methods"},{"location":"user_guide/preprocessing/DenseTransformer/","text":"DenseTransformer: Transforms a sparse into a dense NumPy array, e.g., in a scikit-learn pipeline A simple transformer that converts a sparse into a dense numpy array, e.g., required for scikit-learn's Pipeline when, for example, CountVectorizers are used in combination with estimators that are not compatible with sparse matrices. from mlxtend.preprocessing import DenseTransformer Example 1 from sklearn.pipeline import Pipeline from sklearn.model_selection import GridSearchCV from sklearn.ensemble import RandomForestClassifier from sklearn.feature_extraction.text import CountVectorizer from mlxtend.preprocessing import DenseTransformer import re import numpy as np X_train = np.array(['abc def ghi', 'this is a test', 'this is a test', 'this is a test']) y_train = np.array([0, 0, 1, 1]) pipe_1 = Pipeline([ ('vect', CountVectorizer()), ('to_dense', DenseTransformer()), ('clf', RandomForestClassifier()) ]) parameters_1 = dict( clf__n_estimators=[50, 100, 200], clf__max_features=['sqrt', 'log2', None],) grid_search_1 = GridSearchCV(pipe_1, parameters_1, n_jobs=1, verbose=1, scoring='accuracy', cv=2) print(\"Performing grid search...\") print(\"pipeline:\", [name for name, _ in pipe_1.steps]) print(\"parameters:\") grid_search_1.fit(X_train, y_train) print(\"Best score: %0.3f\" % grid_search_1.best_score_) print(\"Best parameters set:\") best_parameters_1 = grid_search_1.best_estimator_.get_params() for param_name in sorted(parameters_1.keys()): print(\"\\t%s: %r\" % (param_name, best_parameters_1[param_name])) Performing grid search... pipeline: ['vect', 'to_dense', 'clf'] parameters: Fitting 2 folds for each of 9 candidates, totalling 18 fits Best score: 0.500 Best parameters set: clf__max_features: 'sqrt' clf__n_estimators: 50 [Parallel(n_jobs=1)]: Done 18 out of 18 | elapsed: 3.9s finished API DenseTransformer(return_copy=True) Convert a sparse array into a dense array. For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/preprocessing/DenseTransformer/ Methods fit(X, y=None) Mock method. Does nothing. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] (default: None) Returns self fit_transform(X, y=None) Return a dense version of the input array. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] (default: None) Returns X_dense : dense version of the input X array. get_params(deep=True) Get parameters for this estimator. Parameters deep : boolean, optional If True, will return the parameters for this estimator and contained subobjects that are estimators. Returns params : mapping of string to any Parameter names mapped to their values. set_params( params) Set the parameters of this estimator. The method works on simple estimators as well as on nested objects (such as pipelines). The latter have parameters of the form <component>__<parameter> so that it's possible to update each component of a nested object. Returns self transform(X, y=None) Return a dense version of the input array. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] (default: None) Returns X_dense : dense version of the input X array.","title":"DenseTransformer: Transforms a sparse into a dense NumPy array, e.g., in a scikit-learn pipeline"},{"location":"user_guide/preprocessing/DenseTransformer/#densetransformer-transforms-a-sparse-into-a-dense-numpy-array-eg-in-a-scikit-learn-pipeline","text":"A simple transformer that converts a sparse into a dense numpy array, e.g., required for scikit-learn's Pipeline when, for example, CountVectorizers are used in combination with estimators that are not compatible with sparse matrices. from mlxtend.preprocessing import DenseTransformer","title":"DenseTransformer: Transforms a sparse into a dense NumPy array, e.g., in a scikit-learn pipeline"},{"location":"user_guide/preprocessing/DenseTransformer/#example-1","text":"from sklearn.pipeline import Pipeline from sklearn.model_selection import GridSearchCV from sklearn.ensemble import RandomForestClassifier from sklearn.feature_extraction.text import CountVectorizer from mlxtend.preprocessing import DenseTransformer import re import numpy as np X_train = np.array(['abc def ghi', 'this is a test', 'this is a test', 'this is a test']) y_train = np.array([0, 0, 1, 1]) pipe_1 = Pipeline([ ('vect', CountVectorizer()), ('to_dense', DenseTransformer()), ('clf', RandomForestClassifier()) ]) parameters_1 = dict( clf__n_estimators=[50, 100, 200], clf__max_features=['sqrt', 'log2', None],) grid_search_1 = GridSearchCV(pipe_1, parameters_1, n_jobs=1, verbose=1, scoring='accuracy', cv=2) print(\"Performing grid search...\") print(\"pipeline:\", [name for name, _ in pipe_1.steps]) print(\"parameters:\") grid_search_1.fit(X_train, y_train) print(\"Best score: %0.3f\" % grid_search_1.best_score_) print(\"Best parameters set:\") best_parameters_1 = grid_search_1.best_estimator_.get_params() for param_name in sorted(parameters_1.keys()): print(\"\\t%s: %r\" % (param_name, best_parameters_1[param_name])) Performing grid search... pipeline: ['vect', 'to_dense', 'clf'] parameters: Fitting 2 folds for each of 9 candidates, totalling 18 fits Best score: 0.500 Best parameters set: clf__max_features: 'sqrt' clf__n_estimators: 50 [Parallel(n_jobs=1)]: Done 18 out of 18 | elapsed: 3.9s finished","title":"Example 1"},{"location":"user_guide/preprocessing/DenseTransformer/#api","text":"DenseTransformer(return_copy=True) Convert a sparse array into a dense array. For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/preprocessing/DenseTransformer/","title":"API"},{"location":"user_guide/preprocessing/DenseTransformer/#methods","text":"fit(X, y=None) Mock method. Does nothing. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] (default: None) Returns self fit_transform(X, y=None) Return a dense version of the input array. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] (default: None) Returns X_dense : dense version of the input X array. get_params(deep=True) Get parameters for this estimator. Parameters deep : boolean, optional If True, will return the parameters for this estimator and contained subobjects that are estimators. Returns params : mapping of string to any Parameter names mapped to their values. set_params( params) Set the parameters of this estimator. The method works on simple estimators as well as on nested objects (such as pipelines). The latter have parameters of the form <component>__<parameter> so that it's possible to update each component of a nested object. Returns self transform(X, y=None) Return a dense version of the input array. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] (default: None) Returns X_dense : dense version of the input X array.","title":"Methods"},{"location":"user_guide/preprocessing/MeanCenterer/","text":"MeanCenterer: column-based mean centering on a NumPy array A transformer object that performs column-based mean centering on a NumPy array. from mlxtend.preprocessing import MeanCenterer Example 1 - Centering a NumPy Array Use the fit method to fit the column means of a dataset (e.g., the training dataset) to a new MeanCenterer object. Then, call the transform method on the same dataset to center it at the sample mean. import numpy as np from mlxtend.preprocessing import MeanCenterer X_train = np.array( [[1, 2, 3], [4, 5, 6], [7, 8, 9]]) mc = MeanCenterer().fit(X_train) mc.transform(X_train) array([[-3., -3., -3.], [ 0., 0., 0.], [ 3., 3., 3.]]) API MeanCenterer() Column centering of vectors and matrices. Attributes col_means : numpy.ndarray [n_columns] NumPy array storing the mean values for centering after fitting the MeanCenterer object. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/preprocessing/MeanCenterer/ Methods fit(X) Gets the column means for mean centering. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Array of data vectors, where n_samples is the number of samples and n_features is the number of features. Returns self fit_transform(X) Fits and transforms an arry. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Array of data vectors, where n_samples is the number of samples and n_features is the number of features. Returns X_tr : {array-like, sparse matrix}, shape = [n_samples, n_features] A copy of the input array with the columns centered. transform(X) Centers a NumPy array. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Array of data vectors, where n_samples is the number of samples and n_features is the number of features. Returns X_tr : {array-like, sparse matrix}, shape = [n_samples, n_features] A copy of the input array with the columns centered. ython","title":"MeanCenterer: column-based mean centering on a NumPy array"},{"location":"user_guide/preprocessing/MeanCenterer/#meancenterer-column-based-mean-centering-on-a-numpy-array","text":"A transformer object that performs column-based mean centering on a NumPy array. from mlxtend.preprocessing import MeanCenterer","title":"MeanCenterer: column-based mean centering on a NumPy array"},{"location":"user_guide/preprocessing/MeanCenterer/#example-1-centering-a-numpy-array","text":"Use the fit method to fit the column means of a dataset (e.g., the training dataset) to a new MeanCenterer object. Then, call the transform method on the same dataset to center it at the sample mean. import numpy as np from mlxtend.preprocessing import MeanCenterer X_train = np.array( [[1, 2, 3], [4, 5, 6], [7, 8, 9]]) mc = MeanCenterer().fit(X_train) mc.transform(X_train) array([[-3., -3., -3.], [ 0., 0., 0.], [ 3., 3., 3.]])","title":"Example 1 - Centering a NumPy Array"},{"location":"user_guide/preprocessing/MeanCenterer/#api","text":"MeanCenterer() Column centering of vectors and matrices. Attributes col_means : numpy.ndarray [n_columns] NumPy array storing the mean values for centering after fitting the MeanCenterer object. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/preprocessing/MeanCenterer/","title":"API"},{"location":"user_guide/preprocessing/MeanCenterer/#methods","text":"fit(X) Gets the column means for mean centering. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Array of data vectors, where n_samples is the number of samples and n_features is the number of features. Returns self fit_transform(X) Fits and transforms an arry. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Array of data vectors, where n_samples is the number of samples and n_features is the number of features. Returns X_tr : {array-like, sparse matrix}, shape = [n_samples, n_features] A copy of the input array with the columns centered. transform(X) Centers a NumPy array. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Array of data vectors, where n_samples is the number of samples and n_features is the number of features. Returns X_tr : {array-like, sparse matrix}, shape = [n_samples, n_features] A copy of the input array with the columns centered. ython","title":"Methods"},{"location":"user_guide/preprocessing/TransactionEncoder/","text":"TransactionEncoder: Convert item lists into transaction data for frequent itemset mining Encoder class for transaction data in Python lists from mlxtend.preprocessing import TransactionEncoder Overview Encodes database transaction data in form of a Python list of lists into a NumPy array. Example 1 Suppose we have the following transaction data: from mlxtend.preprocessing import TransactionEncoder dataset = [['Apple', 'Beer', 'Rice', 'Chicken'], ['Apple', 'Beer', 'Rice'], ['Apple', 'Beer'], ['Apple', 'Bananas'], ['Milk', 'Beer', 'Rice', 'Chicken'], ['Milk', 'Beer', 'Rice'], ['Milk', 'Beer'], ['Apple', 'Bananas']] Using and TransactionEncoder object, we can transform this dataset into an array format suitable for typical machine learning APIs. Via the fit method, the TransactionEncoder learns the unique labels in the dataset, and via the transform method, it transforms the input dataset (a Python list of lists) into a one-hot encoded NumPy boolean array: te = TransactionEncoder() te_ary = te.fit(dataset).transform(dataset) te_ary array([[ True, False, True, True, False, True], [ True, False, True, False, False, True], [ True, False, True, False, False, False], [ True, True, False, False, False, False], [False, False, True, True, True, True], [False, False, True, False, True, True], [False, False, True, False, True, False], [ True, True, False, False, False, False]], dtype=bool) The NumPy array is boolean for the sake of memory efficiency when working with large datasets. If a classic integer representation is desired instead, we can just convert the array to the appropriate type: te_ary.astype(\"int\") array([[1, 0, 1, 1, 0, 1], [1, 0, 1, 0, 0, 1], [1, 0, 1, 0, 0, 0], [1, 1, 0, 0, 0, 0], [0, 0, 1, 1, 1, 1], [0, 0, 1, 0, 1, 1], [0, 0, 1, 0, 1, 0], [1, 1, 0, 0, 0, 0]]) After fitting, the unique column names that correspond to the data array shown above can be accessed via the columns_ attribute: te.columns_ ['Apple', 'Bananas', 'Beer', 'Chicken', 'Milk', 'Rice'] For our convenience, we can turn theencoded array into a pandas DataFrame : import pandas as pd pd.DataFrame(te_ary, columns=te.columns_) .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } Apple Bananas Beer Chicken Milk Rice 0 True False True True False True 1 True False True False False True 2 True False True False False False 3 True True False False False False 4 False False True True True True 5 False False True False True True 6 False False True False True False 7 True True False False False False If we desire, we can turn the one-hot encoded array back into a transaction list of lists via the inverse_transform function: first4 = te_ary[:4] te.inverse_transform(first4) [['Apple', 'Beer', 'Chicken', 'Rice'], ['Apple', 'Beer', 'Rice'], ['Apple', 'Beer'], ['Apple', 'Bananas']] API TransactionEncoder() Encoder class for transaction data in Python lists Parameters None Attributes columns_: list List of unique names in the X input list of lists Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/preprocessing/TransactionEncoder/ Methods fit(X) Learn unique column names from transaction DataFrame Parameters X : list of lists A python list of lists, where the outer list stores the n transactions and the inner list stores the items in each transaction. For example, [['Apple', 'Beer', 'Rice', 'Chicken'], ['Apple', 'Beer', 'Rice'], ['Apple', 'Beer'], ['Apple', 'Bananas'], ['Milk', 'Beer', 'Rice', 'Chicken'], ['Milk', 'Beer', 'Rice'], ['Milk', 'Beer'], ['Apple', 'Bananas']] fit_transform(X, sparse=False) Fit a TransactionEncoder encoder and transform a dataset. get_params(deep=True) Get parameters for this estimator. Parameters deep : boolean, optional If True, will return the parameters for this estimator and contained subobjects that are estimators. Returns params : mapping of string to any Parameter names mapped to their values. inverse_transform(array) Transforms an encoded NumPy array back into transactions. Parameters array : NumPy array [n_transactions, n_unique_items] The NumPy one-hot encoded boolean array of the input transactions, where the columns represent the unique items found in the input array in alphabetic order For example, array([[True , False, True , True , False, True ], [True , False, True , False, False, True ], [True , False, True , False, False, False], [True , True , False, False, False, False], [False, False, True , True , True , True ], [False, False, True , False, True , True ], [False, False, True , False, True , False], [True , True , False, False, False, False]]) The corresponding column labels are available as self.columns_, e.g., ['Apple', 'Bananas', 'Beer', 'Chicken', 'Milk', 'Rice'] Returns X : list of lists A python list of lists, where the outer list stores the n transactions and the inner list stores the items in each transaction. For example, [['Apple', 'Beer', 'Rice', 'Chicken'], ['Apple', 'Beer', 'Rice'], ['Apple', 'Beer'], ['Apple', 'Bananas'], ['Milk', 'Beer', 'Rice', 'Chicken'], ['Milk', 'Beer', 'Rice'], ['Milk', 'Beer'], ['Apple', 'Bananas']] set_params( params) Set the parameters of this estimator. The method works on simple estimators as well as on nested objects (such as pipelines). The latter have parameters of the form <component>__<parameter> so that it's possible to update each component of a nested object. Returns self transform(X, sparse=False) Transform transactions into a one-hot encoded NumPy array. Parameters X : list of lists A python list of lists, where the outer list stores the n transactions and the inner list stores the items in each transaction. For example, [['Apple', 'Beer', 'Rice', 'Chicken'], ['Apple', 'Beer', 'Rice'], ['Apple', 'Beer'], ['Apple', 'Bananas'], ['Milk', 'Beer', 'Rice', 'Chicken'], ['Milk', 'Beer', 'Rice'], ['Milk', 'Beer'], ['Apple', 'Bananas']] sparse: bool (default=False) If True, transform will return Compressed Sparse Row matrix instead of the regular one. Returns array : NumPy array [n_transactions, n_unique_items] if sparse=False (default). Compressed Sparse Row matrix otherwise The one-hot encoded boolean array of the input transactions, where the columns represent the unique items found in the input array in alphabetic order. Exact representation depends on the sparse argument For example, array([[True , False, True , True , False, True ], [True , False, True , False, False, True ], [True , False, True , False, False, False], [True , True , False, False, False, False], [False, False, True , True , True , True ], [False, False, True , False, True , True ], [False, False, True , False, True , False], [True , True , False, False, False, False]]) The corresponding column labels are available as self.columns_, e.g., ['Apple', 'Bananas', 'Beer', 'Chicken', 'Milk', 'Rice'] ython","title":"TransactionEncoder"},{"location":"user_guide/preprocessing/TransactionEncoder/#transactionencoder-convert-item-lists-into-transaction-data-for-frequent-itemset-mining","text":"Encoder class for transaction data in Python lists from mlxtend.preprocessing import TransactionEncoder","title":"TransactionEncoder: Convert item lists into transaction data for frequent itemset mining"},{"location":"user_guide/preprocessing/TransactionEncoder/#overview","text":"Encodes database transaction data in form of a Python list of lists into a NumPy array.","title":"Overview"},{"location":"user_guide/preprocessing/TransactionEncoder/#example-1","text":"Suppose we have the following transaction data: from mlxtend.preprocessing import TransactionEncoder dataset = [['Apple', 'Beer', 'Rice', 'Chicken'], ['Apple', 'Beer', 'Rice'], ['Apple', 'Beer'], ['Apple', 'Bananas'], ['Milk', 'Beer', 'Rice', 'Chicken'], ['Milk', 'Beer', 'Rice'], ['Milk', 'Beer'], ['Apple', 'Bananas']] Using and TransactionEncoder object, we can transform this dataset into an array format suitable for typical machine learning APIs. Via the fit method, the TransactionEncoder learns the unique labels in the dataset, and via the transform method, it transforms the input dataset (a Python list of lists) into a one-hot encoded NumPy boolean array: te = TransactionEncoder() te_ary = te.fit(dataset).transform(dataset) te_ary array([[ True, False, True, True, False, True], [ True, False, True, False, False, True], [ True, False, True, False, False, False], [ True, True, False, False, False, False], [False, False, True, True, True, True], [False, False, True, False, True, True], [False, False, True, False, True, False], [ True, True, False, False, False, False]], dtype=bool) The NumPy array is boolean for the sake of memory efficiency when working with large datasets. If a classic integer representation is desired instead, we can just convert the array to the appropriate type: te_ary.astype(\"int\") array([[1, 0, 1, 1, 0, 1], [1, 0, 1, 0, 0, 1], [1, 0, 1, 0, 0, 0], [1, 1, 0, 0, 0, 0], [0, 0, 1, 1, 1, 1], [0, 0, 1, 0, 1, 1], [0, 0, 1, 0, 1, 0], [1, 1, 0, 0, 0, 0]]) After fitting, the unique column names that correspond to the data array shown above can be accessed via the columns_ attribute: te.columns_ ['Apple', 'Bananas', 'Beer', 'Chicken', 'Milk', 'Rice'] For our convenience, we can turn theencoded array into a pandas DataFrame : import pandas as pd pd.DataFrame(te_ary, columns=te.columns_) .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } Apple Bananas Beer Chicken Milk Rice 0 True False True True False True 1 True False True False False True 2 True False True False False False 3 True True False False False False 4 False False True True True True 5 False False True False True True 6 False False True False True False 7 True True False False False False If we desire, we can turn the one-hot encoded array back into a transaction list of lists via the inverse_transform function: first4 = te_ary[:4] te.inverse_transform(first4) [['Apple', 'Beer', 'Chicken', 'Rice'], ['Apple', 'Beer', 'Rice'], ['Apple', 'Beer'], ['Apple', 'Bananas']]","title":"Example 1"},{"location":"user_guide/preprocessing/TransactionEncoder/#api","text":"TransactionEncoder() Encoder class for transaction data in Python lists Parameters None Attributes columns_: list List of unique names in the X input list of lists Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/preprocessing/TransactionEncoder/","title":"API"},{"location":"user_guide/preprocessing/TransactionEncoder/#methods","text":"fit(X) Learn unique column names from transaction DataFrame Parameters X : list of lists A python list of lists, where the outer list stores the n transactions and the inner list stores the items in each transaction. For example, [['Apple', 'Beer', 'Rice', 'Chicken'], ['Apple', 'Beer', 'Rice'], ['Apple', 'Beer'], ['Apple', 'Bananas'], ['Milk', 'Beer', 'Rice', 'Chicken'], ['Milk', 'Beer', 'Rice'], ['Milk', 'Beer'], ['Apple', 'Bananas']] fit_transform(X, sparse=False) Fit a TransactionEncoder encoder and transform a dataset. get_params(deep=True) Get parameters for this estimator. Parameters deep : boolean, optional If True, will return the parameters for this estimator and contained subobjects that are estimators. Returns params : mapping of string to any Parameter names mapped to their values. inverse_transform(array) Transforms an encoded NumPy array back into transactions. Parameters array : NumPy array [n_transactions, n_unique_items] The NumPy one-hot encoded boolean array of the input transactions, where the columns represent the unique items found in the input array in alphabetic order For example, array([[True , False, True , True , False, True ], [True , False, True , False, False, True ], [True , False, True , False, False, False], [True , True , False, False, False, False], [False, False, True , True , True , True ], [False, False, True , False, True , True ], [False, False, True , False, True , False], [True , True , False, False, False, False]]) The corresponding column labels are available as self.columns_, e.g., ['Apple', 'Bananas', 'Beer', 'Chicken', 'Milk', 'Rice'] Returns X : list of lists A python list of lists, where the outer list stores the n transactions and the inner list stores the items in each transaction. For example, [['Apple', 'Beer', 'Rice', 'Chicken'], ['Apple', 'Beer', 'Rice'], ['Apple', 'Beer'], ['Apple', 'Bananas'], ['Milk', 'Beer', 'Rice', 'Chicken'], ['Milk', 'Beer', 'Rice'], ['Milk', 'Beer'], ['Apple', 'Bananas']] set_params( params) Set the parameters of this estimator. The method works on simple estimators as well as on nested objects (such as pipelines). The latter have parameters of the form <component>__<parameter> so that it's possible to update each component of a nested object. Returns self transform(X, sparse=False) Transform transactions into a one-hot encoded NumPy array. Parameters X : list of lists A python list of lists, where the outer list stores the n transactions and the inner list stores the items in each transaction. For example, [['Apple', 'Beer', 'Rice', 'Chicken'], ['Apple', 'Beer', 'Rice'], ['Apple', 'Beer'], ['Apple', 'Bananas'], ['Milk', 'Beer', 'Rice', 'Chicken'], ['Milk', 'Beer', 'Rice'], ['Milk', 'Beer'], ['Apple', 'Bananas']] sparse: bool (default=False) If True, transform will return Compressed Sparse Row matrix instead of the regular one. Returns array : NumPy array [n_transactions, n_unique_items] if sparse=False (default). Compressed Sparse Row matrix otherwise The one-hot encoded boolean array of the input transactions, where the columns represent the unique items found in the input array in alphabetic order. Exact representation depends on the sparse argument For example, array([[True , False, True , True , False, True ], [True , False, True , False, False, True ], [True , False, True , False, False, False], [True , True , False, False, False, False], [False, False, True , True , True , True ], [False, False, True , False, True , True ], [False, False, True , False, True , False], [True , True , False, False, False, False]]) The corresponding column labels are available as self.columns_, e.g., ['Apple', 'Bananas', 'Beer', 'Chicken', 'Milk', 'Rice'] ython","title":"Methods"},{"location":"user_guide/preprocessing/minmax_scaling/","text":"MinMaxScaling: Min-max scaling fpr pandas DataFrames and NumPy arrays A function for min-max scaling of pandas DataFrames or NumPy arrays. from mlxtend.preprocessing import MinMaxScaling An alternative approach to Z-score normalization (or standardization) is the so-called Min-Max scaling (often also simply called \"normalization\" - a common cause for ambiguities). In this approach, the data is scaled to a fixed range - usually 0 to 1. The cost of having this bounded range - in contrast to standardization - is that we will end up with smaller standard deviations, which can suppress the effect of outliers. A Min-Max scaling is typically done via the following equation: X_{sc} = \\frac{X - X_{min}}{X_{max} - X_{min}}. One family of algorithms that is scale-invariant encompasses tree-based learning algorithms. Let's take the general CART decision tree algorithm. Without going into much depth regarding information gain and impurity measures, we can think of the decision as \"is feature x_i >= some_val?\" Intuitively, we can see that it really doesn't matter on which scale this feature is (centimeters, Fahrenheit, a standardized scale -- it really doesn't matter). Some examples of algorithms where feature scaling matters are: k-nearest neighbors with an Euclidean distance measure if want all features to contribute equally k-means (see k-nearest neighbors) logistic regression, SVMs, perceptrons, neural networks etc. if you are using gradient descent/ascent-based optimization, otherwise some weights will update much faster than others linear discriminant analysis, principal component analysis, kernel principal component analysis since you want to find directions of maximizing the variance (under the constraints that those directions/eigenvectors/principal components are orthogonal); you want to have features on the same scale since you'd emphasize variables on \"larger measurement scales\" more. There are many more cases than I can possibly list here ... I always recommend you to think about the algorithm and what it's doing, and then it typically becomes obvious whether we want to scale your features or not. In addition, we'd also want to think about whether we want to \"standardize\" or \"normalize\" (here: scaling to [0, 1] range) our data. Some algorithms assume that our data is centered at 0. For example, if we initialize the weights of a small multi-layer perceptron with tanh activation units to 0 or small random values centered around zero, we want to update the model weights \"equally.\" As a rule of thumb I'd say: When in doubt, just standardize the data, it shouldn't hurt. Example 1 - Scaling a Pandas DataFrame import pandas as pd s1 = pd.Series([1, 2, 3, 4, 5, 6], index=(range(6))) s2 = pd.Series([10, 9, 8, 7, 6, 5], index=(range(6))) df = pd.DataFrame(s1, columns=['s1']) df['s2'] = s2 df .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } s1 s2 0 1 10 1 2 9 2 3 8 3 4 7 4 5 6 5 6 5 from mlxtend.preprocessing import minmax_scaling minmax_scaling(df, columns=['s1', 's2']) .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } s1 s2 0 0.0 1.0 1 0.2 0.8 2 0.4 0.6 3 0.6 0.4 4 0.8 0.2 5 1.0 0.0 Example 2 - Scaling a NumPy Array import numpy as np X = np.array([[1, 10], [2, 9], [3, 8], [4, 7], [5, 6], [6, 5]]) X array([[ 1, 10], [ 2, 9], [ 3, 8], [ 4, 7], [ 5, 6], [ 6, 5]]) from mlxtend.preprocessing import minmax_scaling minmax_scaling(X, columns=[0, 1]) array([[0. , 1. ], [0.2, 0.8], [0.4, 0.6], [0.6, 0.4], [0.8, 0.2], [1. , 0. ]]) API minmax_scaling(array, columns, min_val=0, max_val=1) Min max scaling of pandas' DataFrames. Parameters array : pandas DataFrame or NumPy ndarray, shape = [n_rows, n_columns]. columns : array-like, shape = [n_columns] Array-like with column names, e.g., ['col1', 'col2', ...] or column indices [0, 2, 4, ...] min_val : int or float , optional (default= 0 ) minimum value after rescaling. max_val : int or float , optional (default= 1 ) maximum value after rescaling. Returns df_new : pandas DataFrame object. Copy of the array or DataFrame with rescaled columns. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/preprocessing/minmax_scaling/","title":"MinMaxScaling: Min-max scaling fpr pandas DataFrames and NumPy arrays"},{"location":"user_guide/preprocessing/minmax_scaling/#minmaxscaling-min-max-scaling-fpr-pandas-dataframes-and-numpy-arrays","text":"A function for min-max scaling of pandas DataFrames or NumPy arrays. from mlxtend.preprocessing import MinMaxScaling An alternative approach to Z-score normalization (or standardization) is the so-called Min-Max scaling (often also simply called \"normalization\" - a common cause for ambiguities). In this approach, the data is scaled to a fixed range - usually 0 to 1. The cost of having this bounded range - in contrast to standardization - is that we will end up with smaller standard deviations, which can suppress the effect of outliers. A Min-Max scaling is typically done via the following equation: X_{sc} = \\frac{X - X_{min}}{X_{max} - X_{min}}. One family of algorithms that is scale-invariant encompasses tree-based learning algorithms. Let's take the general CART decision tree algorithm. Without going into much depth regarding information gain and impurity measures, we can think of the decision as \"is feature x_i >= some_val?\" Intuitively, we can see that it really doesn't matter on which scale this feature is (centimeters, Fahrenheit, a standardized scale -- it really doesn't matter). Some examples of algorithms where feature scaling matters are: k-nearest neighbors with an Euclidean distance measure if want all features to contribute equally k-means (see k-nearest neighbors) logistic regression, SVMs, perceptrons, neural networks etc. if you are using gradient descent/ascent-based optimization, otherwise some weights will update much faster than others linear discriminant analysis, principal component analysis, kernel principal component analysis since you want to find directions of maximizing the variance (under the constraints that those directions/eigenvectors/principal components are orthogonal); you want to have features on the same scale since you'd emphasize variables on \"larger measurement scales\" more. There are many more cases than I can possibly list here ... I always recommend you to think about the algorithm and what it's doing, and then it typically becomes obvious whether we want to scale your features or not. In addition, we'd also want to think about whether we want to \"standardize\" or \"normalize\" (here: scaling to [0, 1] range) our data. Some algorithms assume that our data is centered at 0. For example, if we initialize the weights of a small multi-layer perceptron with tanh activation units to 0 or small random values centered around zero, we want to update the model weights \"equally.\" As a rule of thumb I'd say: When in doubt, just standardize the data, it shouldn't hurt.","title":"MinMaxScaling: Min-max scaling fpr pandas DataFrames and NumPy arrays"},{"location":"user_guide/preprocessing/minmax_scaling/#example-1-scaling-a-pandas-dataframe","text":"import pandas as pd s1 = pd.Series([1, 2, 3, 4, 5, 6], index=(range(6))) s2 = pd.Series([10, 9, 8, 7, 6, 5], index=(range(6))) df = pd.DataFrame(s1, columns=['s1']) df['s2'] = s2 df .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } s1 s2 0 1 10 1 2 9 2 3 8 3 4 7 4 5 6 5 6 5 from mlxtend.preprocessing import minmax_scaling minmax_scaling(df, columns=['s1', 's2']) .dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; } s1 s2 0 0.0 1.0 1 0.2 0.8 2 0.4 0.6 3 0.6 0.4 4 0.8 0.2 5 1.0 0.0","title":"Example 1 - Scaling a Pandas DataFrame"},{"location":"user_guide/preprocessing/minmax_scaling/#example-2-scaling-a-numpy-array","text":"import numpy as np X = np.array([[1, 10], [2, 9], [3, 8], [4, 7], [5, 6], [6, 5]]) X array([[ 1, 10], [ 2, 9], [ 3, 8], [ 4, 7], [ 5, 6], [ 6, 5]]) from mlxtend.preprocessing import minmax_scaling minmax_scaling(X, columns=[0, 1]) array([[0. , 1. ], [0.2, 0.8], [0.4, 0.6], [0.6, 0.4], [0.8, 0.2], [1. , 0. ]])","title":"Example 2 - Scaling a NumPy Array"},{"location":"user_guide/preprocessing/minmax_scaling/#api","text":"minmax_scaling(array, columns, min_val=0, max_val=1) Min max scaling of pandas' DataFrames. Parameters array : pandas DataFrame or NumPy ndarray, shape = [n_rows, n_columns]. columns : array-like, shape = [n_columns] Array-like with column names, e.g., ['col1', 'col2', ...] or column indices [0, 2, 4, ...] min_val : int or float , optional (default= 0 ) minimum value after rescaling. max_val : int or float , optional (default= 1 ) maximum value after rescaling. Returns df_new : pandas DataFrame object. Copy of the array or DataFrame with rescaled columns. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/preprocessing/minmax_scaling/","title":"API"},{"location":"user_guide/preprocessing/one-hot_encoding/","text":"one_hot: One-Hot encoding function for class label arrays A function that performs one-hot encoding for class labels. from mlxtend.preprocessing import one_hot Overview Typical supervised machine learning algorithms for classifications assume that the class labels are nominal (a special case of categorical where no order is implied). A typical example of an nominal feature would be \"color\" since we can't say (in most applications) that \"orange > blue > red\". The one_hot function provides a simple interface to convert class label integers into a so-called one-hot array, where each unique label is represented as a column in the new array. For example, let's assume we have 5 data points from 3 different classes: 0, 1, and 2. y = [0, # sample 1, class 0 1, # sample 2, class 1 0, # sample 3, class 0 2, # sample 4, class 2 2] # sample 5, class 2 After one-hot encoding, we then obtain the following array (note that the index position of the \"1\" in each row denotes the class label of this sample): y = [[1, 0, 0], # sample 1, class 0 [0, 1, 0], # sample 2, class 1 [1, 0, 0], # sample 3, class 0 [0, 0, 1], # sample 4, class 2 [0, 0, 1] # sample 5, class 2 ]) Example 1 - Defaults from mlxtend.preprocessing import one_hot import numpy as np y = np.array([0, 1, 2, 1, 2]) one_hot(y) array([[ 1., 0., 0.], [ 0., 1., 0.], [ 0., 0., 1.], [ 0., 1., 0.], [ 0., 0., 1.]]) Example 2 - Python Lists from mlxtend.preprocessing import one_hot y = [0, 1, 2, 1, 2] one_hot(y) array([[ 1., 0., 0.], [ 0., 1., 0.], [ 0., 0., 1.], [ 0., 1., 0.], [ 0., 0., 1.]]) Example 3 - Integer Arrays from mlxtend.preprocessing import one_hot y = [0, 1, 2, 1, 2] one_hot(y, dtype='int') array([[1, 0, 0], [0, 1, 0], [0, 0, 1], [0, 1, 0], [0, 0, 1]]) Example 4 - Arbitrary Numbers of Class Labels from mlxtend.preprocessing import one_hot y = [0, 1, 2, 1, 2] one_hot(y, num_labels=10) array([[ 1., 0., 0., 0., 0., 0., 0., 0., 0., 0.], [ 0., 1., 0., 0., 0., 0., 0., 0., 0., 0.], [ 0., 0., 1., 0., 0., 0., 0., 0., 0., 0.], [ 0., 1., 0., 0., 0., 0., 0., 0., 0., 0.], [ 0., 0., 1., 0., 0., 0., 0., 0., 0., 0.]]) API one_hot(y, num_labels='auto', dtype='float') One-hot encoding of class labels Parameters y : array-like, shape = [n_classlabels] Python list or numpy array consisting of class labels. num_labels : int or 'auto' Number of unique labels in the class label array. Infers the number of unique labels from the input array if set to 'auto'. dtype : str NumPy array type (float, float32, float64) of the output array. Returns ary : numpy.ndarray, shape = [n_classlabels] One-hot encoded array, where each sample is represented as a row vector in the returned array. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/preprocessing/one_hot/","title":"One hot encoding"},{"location":"user_guide/preprocessing/one-hot_encoding/#one_hot-one-hot-encoding-function-for-class-label-arrays","text":"A function that performs one-hot encoding for class labels. from mlxtend.preprocessing import one_hot","title":"one_hot: One-Hot encoding function for class label arrays"},{"location":"user_guide/preprocessing/one-hot_encoding/#overview","text":"Typical supervised machine learning algorithms for classifications assume that the class labels are nominal (a special case of categorical where no order is implied). A typical example of an nominal feature would be \"color\" since we can't say (in most applications) that \"orange > blue > red\". The one_hot function provides a simple interface to convert class label integers into a so-called one-hot array, where each unique label is represented as a column in the new array. For example, let's assume we have 5 data points from 3 different classes: 0, 1, and 2. y = [0, # sample 1, class 0 1, # sample 2, class 1 0, # sample 3, class 0 2, # sample 4, class 2 2] # sample 5, class 2 After one-hot encoding, we then obtain the following array (note that the index position of the \"1\" in each row denotes the class label of this sample): y = [[1, 0, 0], # sample 1, class 0 [0, 1, 0], # sample 2, class 1 [1, 0, 0], # sample 3, class 0 [0, 0, 1], # sample 4, class 2 [0, 0, 1] # sample 5, class 2 ])","title":"Overview"},{"location":"user_guide/preprocessing/one-hot_encoding/#example-1-defaults","text":"from mlxtend.preprocessing import one_hot import numpy as np y = np.array([0, 1, 2, 1, 2]) one_hot(y) array([[ 1., 0., 0.], [ 0., 1., 0.], [ 0., 0., 1.], [ 0., 1., 0.], [ 0., 0., 1.]])","title":"Example 1 - Defaults"},{"location":"user_guide/preprocessing/one-hot_encoding/#example-2-python-lists","text":"from mlxtend.preprocessing import one_hot y = [0, 1, 2, 1, 2] one_hot(y) array([[ 1., 0., 0.], [ 0., 1., 0.], [ 0., 0., 1.], [ 0., 1., 0.], [ 0., 0., 1.]])","title":"Example 2 - Python Lists"},{"location":"user_guide/preprocessing/one-hot_encoding/#example-3-integer-arrays","text":"from mlxtend.preprocessing import one_hot y = [0, 1, 2, 1, 2] one_hot(y, dtype='int') array([[1, 0, 0], [0, 1, 0], [0, 0, 1], [0, 1, 0], [0, 0, 1]])","title":"Example 3 - Integer Arrays"},{"location":"user_guide/preprocessing/one-hot_encoding/#example-4-arbitrary-numbers-of-class-labels","text":"from mlxtend.preprocessing import one_hot y = [0, 1, 2, 1, 2] one_hot(y, num_labels=10) array([[ 1., 0., 0., 0., 0., 0., 0., 0., 0., 0.], [ 0., 1., 0., 0., 0., 0., 0., 0., 0., 0.], [ 0., 0., 1., 0., 0., 0., 0., 0., 0., 0.], [ 0., 1., 0., 0., 0., 0., 0., 0., 0., 0.], [ 0., 0., 1., 0., 0., 0., 0., 0., 0., 0.]])","title":"Example 4 - Arbitrary Numbers of Class Labels"},{"location":"user_guide/preprocessing/one-hot_encoding/#api","text":"one_hot(y, num_labels='auto', dtype='float') One-hot encoding of class labels Parameters y : array-like, shape = [n_classlabels] Python list or numpy array consisting of class labels. num_labels : int or 'auto' Number of unique labels in the class label array. Infers the number of unique labels from the input array if set to 'auto'. dtype : str NumPy array type (float, float32, float64) of the output array. Returns ary : numpy.ndarray, shape = [n_classlabels] One-hot encoded array, where each sample is represented as a row vector in the returned array. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/preprocessing/one_hot/","title":"API"},{"location":"user_guide/preprocessing/shuffle_arrays_unison/","text":"shuffle_arrays_unison: shuffle arrays in a consistent fashion A function for NumPy arrays in unison. from mlxtend.preprocessing import shuffle_arrays_unison Example 1 - Scaling a Pandas DataFrame import numpy as np from mlxtend.preprocessing import shuffle_arrays_unison X = np.array([[1, 2, 3], [4, 5, 6], [7, 8, 9]]) y = np.array([1, 2, 3]) print('X:\\n%s' % X) print('y:\\n%s' % y) X: [[1 2 3] [4 5 6] [7 8 9]] y: [1 2 3] X2, y2 = shuffle_arrays_unison(arrays=[X, y], random_seed=3) print('X2:\\n%s' % X2) print('y2:\\n%s' % y2) X2: [[4 5 6] [1 2 3] [7 8 9]] y2: [2 1 3] API shuffle_arrays_unison(arrays, random_seed=None) Shuffle NumPy arrays in unison. Parameters arrays : array-like, shape = [n_arrays] A list of NumPy arrays. random_seed : int (default: None) Sets the random state. Returns shuffled_arrays : A list of NumPy arrays after shuffling. Examples >>> import numpy as np >>> from mlxtend.preprocessing import shuffle_arrays_unison >>> X1 = np.array([[1, 2, 3], [4, 5, 6], [7, 8, 9]]) >>> y1 = np.array([1, 2, 3]) >>> X2, y2 = shuffle_arrays_unison(arrays=[X1, y1], random_seed=3) >>> assert(X2.all() == np.array([[4, 5, 6], [1, 2, 3], [7, 8, 9]]).all()) >>> assert(y2.all() == np.array([2, 1, 3]).all()) >>> For more usage examples, please see https://rasbt.github.io/mlxtend/user_guide/preprocessing/shuffle_arrays_unison/","title":"shuffle_arrays_unison: shuffle arrays in a consistent fashion"},{"location":"user_guide/preprocessing/shuffle_arrays_unison/#shuffle_arrays_unison-shuffle-arrays-in-a-consistent-fashion","text":"A function for NumPy arrays in unison. from mlxtend.preprocessing import shuffle_arrays_unison","title":"shuffle_arrays_unison: shuffle arrays in a consistent fashion"},{"location":"user_guide/preprocessing/shuffle_arrays_unison/#example-1-scaling-a-pandas-dataframe","text":"import numpy as np from mlxtend.preprocessing import shuffle_arrays_unison X = np.array([[1, 2, 3], [4, 5, 6], [7, 8, 9]]) y = np.array([1, 2, 3]) print('X:\\n%s' % X) print('y:\\n%s' % y) X: [[1 2 3] [4 5 6] [7 8 9]] y: [1 2 3] X2, y2 = shuffle_arrays_unison(arrays=[X, y], random_seed=3) print('X2:\\n%s' % X2) print('y2:\\n%s' % y2) X2: [[4 5 6] [1 2 3] [7 8 9]] y2: [2 1 3]","title":"Example 1 - Scaling a Pandas DataFrame"},{"location":"user_guide/preprocessing/shuffle_arrays_unison/#api","text":"shuffle_arrays_unison(arrays, random_seed=None) Shuffle NumPy arrays in unison. Parameters arrays : array-like, shape = [n_arrays] A list of NumPy arrays. random_seed : int (default: None) Sets the random state. Returns shuffled_arrays : A list of NumPy arrays after shuffling. Examples >>> import numpy as np >>> from mlxtend.preprocessing import shuffle_arrays_unison >>> X1 = np.array([[1, 2, 3], [4, 5, 6], [7, 8, 9]]) >>> y1 = np.array([1, 2, 3]) >>> X2, y2 = shuffle_arrays_unison(arrays=[X1, y1], random_seed=3) >>> assert(X2.all() == np.array([[4, 5, 6], [1, 2, 3], [7, 8, 9]]).all()) >>> assert(y2.all() == np.array([2, 1, 3]).all()) >>> For more usage examples, please see https://rasbt.github.io/mlxtend/user_guide/preprocessing/shuffle_arrays_unison/","title":"API"},{"location":"user_guide/preprocessing/standardize/","text":"standardize: A function to standardize columns in a 2D NumPy array A function that performs column-based standardization on a NumPy array. from mlxtend.preprocessing import standardize Overview The result of standardization (or Z-score normalization) is that the features will be rescaled so that they'll have the properties of a standard normal distribution with \\mu = 0 and \\sigma = 1 . where \\mu is the mean (average) and \\sigma is the standard deviation from the mean; standard scores (also called z scores) of the samples are calculated as z=\\frac{x-\\mu}{\\sigma}. Standardizing the features so that they are centered around 0 with a standard deviation of 1 is not only important if we are comparing measurements that have different units, but it is also a general requirement for the optimal performance of many machine learning algorithms. One family of algorithms that is scale-invariant encompasses tree-based learning algorithms. Let's take the general CART decision tree algorithm. Without going into much depth regarding information gain and impurity measures, we can think of the decision as \"is feature x_i >= some_val?\" Intuitively, we can see that it really doesn't matter on which scale this feature is (centimeters, Fahrenheit, a standardized scale -- it really doesn't matter). Some examples of algorithms where feature scaling matters are: k-nearest neighbors with an Euclidean distance measure if want all features to contribute equally k-means (see k-nearest neighbors) logistic regression, SVMs, perceptrons, neural networks etc. if you are using gradient descent/ascent-based optimization, otherwise some weights will update much faster than others linear discriminant analysis, principal component analysis, kernel principal component analysis since you want to find directions of maximizing the variance (under the constraints that those directions/eigenvectors/principal components are orthogonal); you want to have features on the same scale since you'd emphasize variables on \"larger measurement scales\" more. There are many more cases than I can possibly list here ... I always recommend you to think about the algorithm and what it's doing, and then it typically becomes obvious whether we want to scale your features or not. In addition, we'd also want to think about whether we want to \"standardize\" or \"normalize\" (here: scaling to [0, 1] range) our data. Some algorithms assume that our data is centered at 0. For example, if we initialize the weights of a small multi-layer perceptron with tanh activation units to 0 or small random values centered around zero, we want to update the model weights \"equally.\" As a rule of thumb I'd say: When in doubt, just standardize the data, it shouldn't hurt. Example 1 - Standardize a Pandas DataFrame import pandas as pd s1 = pd.Series([1, 2, 3, 4, 5, 6], index=(range(6))) s2 = pd.Series([10, 9, 8, 7, 6, 5], index=(range(6))) df = pd.DataFrame(s1, columns=['s1']) df['s2'] = s2 df s1 s2 0 1 10 1 2 9 2 3 8 3 4 7 4 5 6 5 6 5 from mlxtend.preprocessing import standardize standardize(df, columns=['s1', 's2']) s1 s2 0 -1.46385 1.46385 1 -0.87831 0.87831 2 -0.29277 0.29277 3 0.29277 -0.29277 4 0.87831 -0.87831 5 1.46385 -1.46385 Example 2 - Standardize a NumPy Array import numpy as np X = np.array([[1, 10], [2, 9], [3, 8], [4, 7], [5, 6], [6, 5]]) X array([[ 1, 10], [ 2, 9], [ 3, 8], [ 4, 7], [ 5, 6], [ 6, 5]]) from mlxtend.preprocessing import standardize standardize(X, columns=[0, 1]) array([[-1.46385011, 1.46385011], [-0.87831007, 0.87831007], [-0.29277002, 0.29277002], [ 0.29277002, -0.29277002], [ 0.87831007, -0.87831007], [ 1.46385011, -1.46385011]]) Example 3 - Re-using parameters In machine learning contexts, it is desired to re-use the parameters that have been obtained from a training set to scale new, future data (including the independent test set). By setting return_params=True , the standardize function returns a second object, a parameter dictionary containing the column means and standard deviations that can be re-used by feeding it to the params parameter upon function call. import numpy as np from mlxtend.preprocessing import standardize X_train = np.array([[1, 10], [4, 7], [3, 8]]) X_test = np.array([[1, 2], [3, 4], [5, 6]]) X_train_std, params = standardize(X_train, columns=[0, 1], return_params=True) X_train_std array([[-1.33630621, 1.33630621], [ 1.06904497, -1.06904497], [ 0.26726124, -0.26726124]]) params {'avgs': array([ 2.66666667, 8.33333333]), 'stds': array([ 1.24721913, 1.24721913])} X_test_std = standardize(X_test, columns=[0, 1], params=params) X_test_std array([[-1.33630621, -5.0779636 ], [ 0.26726124, -3.47439614], [ 1.87082869, -1.87082869]]) API standardize(array, columns=None, ddof=0, return_params=False, params=None) Standardize columns in pandas DataFrames. Parameters array : pandas DataFrame or NumPy ndarray, shape = [n_rows, n_columns]. columns : array-like, shape = [n_columns] (default: None) Array-like with column names, e.g., ['col1', 'col2', ...] or column indices [0, 2, 4, ...] If None, standardizes all columns. ddof : int (default: 0) Delta Degrees of Freedom. The divisor used in calculations is N - ddof, where N represents the number of elements. return_params : dict (default: False) If set to True, a dictionary is returned in addition to the standardized array. The parameter dictionary contains the column means ('avgs') and standard deviations ('stds') of the individual columns. params : dict (default: None) A dictionary with column means and standard deviations as returned by the standardize function if return_params was set to True. If a params dictionary is provided, the standardize function will use these instead of computing them from the current array. Notes If all values in a given column are the same, these values are all set to 0.0 . The standard deviation in the parameters dictionary is consequently set to 1.0 to avoid dividing by zero. Returns df_new : pandas DataFrame object. Copy of the array or DataFrame with standardized columns. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/preprocessing/standardize/","title":"standardize: A function to standardize columns in a 2D NumPy array"},{"location":"user_guide/preprocessing/standardize/#standardize-a-function-to-standardize-columns-in-a-2d-numpy-array","text":"A function that performs column-based standardization on a NumPy array. from mlxtend.preprocessing import standardize","title":"standardize: A function to standardize columns in a 2D NumPy array"},{"location":"user_guide/preprocessing/standardize/#overview","text":"The result of standardization (or Z-score normalization) is that the features will be rescaled so that they'll have the properties of a standard normal distribution with \\mu = 0 and \\sigma = 1 . where \\mu is the mean (average) and \\sigma is the standard deviation from the mean; standard scores (also called z scores) of the samples are calculated as z=\\frac{x-\\mu}{\\sigma}. Standardizing the features so that they are centered around 0 with a standard deviation of 1 is not only important if we are comparing measurements that have different units, but it is also a general requirement for the optimal performance of many machine learning algorithms. One family of algorithms that is scale-invariant encompasses tree-based learning algorithms. Let's take the general CART decision tree algorithm. Without going into much depth regarding information gain and impurity measures, we can think of the decision as \"is feature x_i >= some_val?\" Intuitively, we can see that it really doesn't matter on which scale this feature is (centimeters, Fahrenheit, a standardized scale -- it really doesn't matter). Some examples of algorithms where feature scaling matters are: k-nearest neighbors with an Euclidean distance measure if want all features to contribute equally k-means (see k-nearest neighbors) logistic regression, SVMs, perceptrons, neural networks etc. if you are using gradient descent/ascent-based optimization, otherwise some weights will update much faster than others linear discriminant analysis, principal component analysis, kernel principal component analysis since you want to find directions of maximizing the variance (under the constraints that those directions/eigenvectors/principal components are orthogonal); you want to have features on the same scale since you'd emphasize variables on \"larger measurement scales\" more. There are many more cases than I can possibly list here ... I always recommend you to think about the algorithm and what it's doing, and then it typically becomes obvious whether we want to scale your features or not. In addition, we'd also want to think about whether we want to \"standardize\" or \"normalize\" (here: scaling to [0, 1] range) our data. Some algorithms assume that our data is centered at 0. For example, if we initialize the weights of a small multi-layer perceptron with tanh activation units to 0 or small random values centered around zero, we want to update the model weights \"equally.\" As a rule of thumb I'd say: When in doubt, just standardize the data, it shouldn't hurt.","title":"Overview"},{"location":"user_guide/preprocessing/standardize/#example-1-standardize-a-pandas-dataframe","text":"import pandas as pd s1 = pd.Series([1, 2, 3, 4, 5, 6], index=(range(6))) s2 = pd.Series([10, 9, 8, 7, 6, 5], index=(range(6))) df = pd.DataFrame(s1, columns=['s1']) df['s2'] = s2 df s1 s2 0 1 10 1 2 9 2 3 8 3 4 7 4 5 6 5 6 5 from mlxtend.preprocessing import standardize standardize(df, columns=['s1', 's2']) s1 s2 0 -1.46385 1.46385 1 -0.87831 0.87831 2 -0.29277 0.29277 3 0.29277 -0.29277 4 0.87831 -0.87831 5 1.46385 -1.46385","title":"Example 1 - Standardize a Pandas DataFrame"},{"location":"user_guide/preprocessing/standardize/#example-2-standardize-a-numpy-array","text":"import numpy as np X = np.array([[1, 10], [2, 9], [3, 8], [4, 7], [5, 6], [6, 5]]) X array([[ 1, 10], [ 2, 9], [ 3, 8], [ 4, 7], [ 5, 6], [ 6, 5]]) from mlxtend.preprocessing import standardize standardize(X, columns=[0, 1]) array([[-1.46385011, 1.46385011], [-0.87831007, 0.87831007], [-0.29277002, 0.29277002], [ 0.29277002, -0.29277002], [ 0.87831007, -0.87831007], [ 1.46385011, -1.46385011]])","title":"Example 2 - Standardize a NumPy Array"},{"location":"user_guide/preprocessing/standardize/#example-3-re-using-parameters","text":"In machine learning contexts, it is desired to re-use the parameters that have been obtained from a training set to scale new, future data (including the independent test set). By setting return_params=True , the standardize function returns a second object, a parameter dictionary containing the column means and standard deviations that can be re-used by feeding it to the params parameter upon function call. import numpy as np from mlxtend.preprocessing import standardize X_train = np.array([[1, 10], [4, 7], [3, 8]]) X_test = np.array([[1, 2], [3, 4], [5, 6]]) X_train_std, params = standardize(X_train, columns=[0, 1], return_params=True) X_train_std array([[-1.33630621, 1.33630621], [ 1.06904497, -1.06904497], [ 0.26726124, -0.26726124]]) params {'avgs': array([ 2.66666667, 8.33333333]), 'stds': array([ 1.24721913, 1.24721913])} X_test_std = standardize(X_test, columns=[0, 1], params=params) X_test_std array([[-1.33630621, -5.0779636 ], [ 0.26726124, -3.47439614], [ 1.87082869, -1.87082869]])","title":"Example 3 - Re-using parameters"},{"location":"user_guide/preprocessing/standardize/#api","text":"standardize(array, columns=None, ddof=0, return_params=False, params=None) Standardize columns in pandas DataFrames. Parameters array : pandas DataFrame or NumPy ndarray, shape = [n_rows, n_columns]. columns : array-like, shape = [n_columns] (default: None) Array-like with column names, e.g., ['col1', 'col2', ...] or column indices [0, 2, 4, ...] If None, standardizes all columns. ddof : int (default: 0) Delta Degrees of Freedom. The divisor used in calculations is N - ddof, where N represents the number of elements. return_params : dict (default: False) If set to True, a dictionary is returned in addition to the standardized array. The parameter dictionary contains the column means ('avgs') and standard deviations ('stds') of the individual columns. params : dict (default: None) A dictionary with column means and standard deviations as returned by the standardize function if return_params was set to True. If a params dictionary is provided, the standardize function will use these instead of computing them from the current array. Notes If all values in a given column are the same, these values are all set to 0.0 . The standard deviation in the parameters dictionary is consequently set to 1.0 to avoid dividing by zero. Returns df_new : pandas DataFrame object. Copy of the array or DataFrame with standardized columns. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/preprocessing/standardize/","title":"API"},{"location":"user_guide/regressor/LinearRegression/","text":"LinearRegression: An implementation of ordinary least-squares linear regression A implementation of Ordinary Least Squares simple and multiple linear regression. from mlxtend.regressor import LinearRegression Overview Illustration of a simple linear regression model: In Ordinary Least Squares (OLS) Linear Regression, our goal is to find the line (or hyperplane) that minimizes the vertical offsets. Or in other words, we define the best-fitting line as the line that minimizes the sum of squared errors (SSE) or mean squared error (MSE) between our target variable (y) and our predicted output over all samples i in our dataset of size n . SSE = \\sum_i \\big(\\text{target}^{(i)} - \\text{output}^{(i)}\\big)^2 MSE = \\frac{1}{n} \\times SSE Now, LinearRegression implements a linear regression model for performing ordinary least squares regression using one of the following five approaches: Normal Equations QR Decomposition Method SVD (Singular Value Decomposition) method Gradient Descent Stochastic Gradient Descent Normal Equations (closed-form solution) The closed-form solution should be preferred for \"smaller\" datasets where calculating (a \"costly\") matrix inverse is not a concern. For very large datasets, or datasets where the inverse of [X^T X] may not exist (the matrix is non-invertible or singular, e.g., in case of perfect multicollinearity), the QR, SVD or gradient descent approaches are to be preferred. The linear function (linear regression model) is defined as: y = w_0x_0 + w_1x_1 + ... + w_mx_m = \\sum_{i=0}^{m} = \\mathbf{w}^T\\mathbf{x} where y is the response variable, \\mathbf{x} is an m -dimensional sample vector, and \\mathbf{w} is the weight vector (vector of coefficients). Note that w_0 represents the y-axis intercept of the model and therefore x_0=1 . Using the closed-form solution (normal equation), we compute the weights of the model as follows: \\mathbf{w} = (\\mathbf{X}^T\\mathbf{X})^{-1}\\mathbf{X}^Ty Stable OLS via QR Factorization The QR decomposition method offers a more numerically stable alternative to the closed-form, analytical solution based on the \"normal equations,\" and it can be used to compute the inverse of large matrices more efficiently. QR decomposition method decomposes given matrix into two matrices for which an inverse can be easily obtained. For instance, a given matrix X \\in \\mathbb{R}^{n \\times m} , the QR decomposition into two matrices is: \\mathbf{X = QR}, where Q \\in \\mathbb{R}^{n \\times m} is an orthonormal matrix, such that Q^\\top Q = QQ^\\top = I . The second matrix R \\in \\mathbf{R}^{m \\times m} is an upper triangular matrix. The weight parameters of the ordinary least squares regression model can then be computed as follows [1]: \\mathbf{w} = \\mathbf{R}^{-1}\\mathbf{Q}^\\top y. Stable OLS via Singular Value Decomposition Another alternative way for obtaining the OLS model weights in a numerically stable fashion is by Singular Value Decomposition (SVD), which is defined as: \\mathbf{X}=\\mathbf{U}\\Sigma \\mathbf{V}^\\top, for a given matrix \\mathbf{X} . Then, it can be shown that the pseudo-inverse of X , X^+ , can be obtained as follows [1]: X^+ = \\mathbf{U} \\mathbf{\\Sigma}^+ V^{\\top}. Note that while \\Sigma is the diagonal matrix consisting of singular values of \\mathbf{X} , \\Sigma^{+} is the diagonal matrix consisting of the reciprocals of the singular values. The model weights can then be computed as follows: \\mathbf{w} = \\mathbf{X}^+ \\mathbf{y}. Please note that this OLS method is computationally most inefficient. However, it is a useful approach when the direct method (normal equations) or QR factorization cannot be applied or the normal equations (via \\mathbf{X}^T \\mathbf{X} ) are ill-conditioned [3]. Gradient Descent (GD) and Stochastic Gradient Descent (SGD) See Gradient Descent and Stochastic Gradient Descent and Deriving the Gradient Descent Rule for Linear Regression and Adaline for details. Random shuffling is implemented as: for one or more epochs randomly shuffle samples in the training set for training sample i compute gradients and perform weight updates References [1] Chapter 3, page 55, Linear Methods for Regression. Trevor Hastie; Robert Tibshirani; Jerome Friedman (2009). The Elements of Statistical Learning: Data Mining, Inference, and Prediction (2nd ed.) . New York: Springer. (ISBN 978\u20130\u2013387\u201384858\u20137) [2] G. Strang, Linear Algebra and Its Applications, 2nd Ed., Orlando, FL, Academic Press, Inc., 1980, pp. 139-142. [3] Douglas Wilhelm Harder. Numerical Analysis for Engineering. Section 4.8, ill-conditioned Matrices Example 1 - Closed Form Solution import numpy as np import matplotlib.pyplot as plt from mlxtend.regressor import LinearRegression X = np.array([ 1.0, 2.1, 3.6, 4.2, 6])[:, np.newaxis] y = np.array([ 1.0, 2.0, 3.0, 4.0, 5.0]) ne_lr = LinearRegression() ne_lr.fit(X, y) print('Intercept: %.2f' % ne_lr.b_) print('Slope: %.2f' % ne_lr.w_[0]) def lin_regplot(X, y, model): plt.scatter(X, y, c='blue') plt.plot(X, model.predict(X), color='red') return lin_regplot(X, y, ne_lr) plt.show() Intercept: 0.25 Slope: 0.81 Example 2 - QR decomposition method import numpy as np import matplotlib.pyplot as plt from mlxtend.regressor import LinearRegression X = np.array([ 1.0, 2.1, 3.6, 4.2, 6])[:, np.newaxis] y = np.array([ 1.0, 2.0, 3.0, 4.0, 5.0]) qr_lr = LinearRegression(method='qr') qr_lr.fit(X, y) print('Intercept: %.2f' % qr_lr.b_) print('Slope: %.2f' % qr_lr.w_[0]) def lin_regplot(X, y, model): plt.scatter(X, y, c='blue') plt.plot(X, model.predict(X), color='red') return lin_regplot(X, y, qr_lr) plt.show() Intercept: 0.25 Slope: 0.81 Example 3 - SVD method import numpy as np import matplotlib.pyplot as plt from mlxtend.regressor import LinearRegression X = np.array([ 1.0, 2.1, 3.6, 4.2, 6])[:, np.newaxis] y = np.array([ 1.0, 2.0, 3.0, 4.0, 5.0]) svd_lr = LinearRegression(method='svd') svd_lr.fit(X, y) print('Intercept: %.2f' %svd_lr.b_) print('Slope: %.2f' % svd_lr.w_[0]) def lin_regplot(X, y, model): plt.scatter(X, y, c='blue') plt.plot(X, model.predict(X), color='red') return lin_regplot(X, y, svd_lr) plt.show() Intercept: 0.25 Slope: 0.81 Example 4 - Gradient Descent import numpy as np import matplotlib.pyplot as plt from mlxtend.regressor import LinearRegression X = np.array([ 1.0, 2.1, 3.6, 4.2, 6])[:, np.newaxis] y = np.array([ 1.0, 2.0, 3.0, 4.0, 5.0]) gd_lr = LinearRegression(method='sgd', eta=0.005, epochs=100, minibatches=1, random_seed=123, print_progress=3) gd_lr.fit(X, y) print('Intercept: %.2f' % gd_lr.b_) print('Slope: %.2f' % gd_lr.w_) def lin_regplot(X, y, model): plt.scatter(X, y, c='blue') plt.plot(X, model.predict(X), color='red') return lin_regplot(X, y, gd_lr) plt.show() Iteration: 100/100 | Cost 0.08 | Elapsed: 0:00:00 | ETA: 0:00:00 Intercept: 0.22 Slope: 0.82 # Visualizing the cost to check for convergence and plotting the linear model: plt.plot(range(1, gd_lr.epochs+1), gd_lr.cost_) plt.xlabel('Epochs') plt.ylabel('Cost') plt.ylim([0, 0.2]) plt.tight_layout() plt.show() Example 5 - Stochastic Gradient Descent import numpy as np import matplotlib.pyplot as plt from mlxtend.regressor import LinearRegression X = np.array([ 1.0, 2.1, 3.6, 4.2, 6])[:, np.newaxis] y = np.array([ 1.0, 2.0, 3.0, 4.0, 5.0]) sgd_lr = LinearRegression(method='sgd', eta=0.01, epochs=100, random_seed=0, minibatches=len(y)) sgd_lr.fit(X, y) print('Intercept: %.2f' % sgd_lr.w_) print('Slope: %.2f' % sgd_lr.b_) def lin_regplot(X, y, model): plt.scatter(X, y, c='blue') plt.plot(X, model.predict(X), color='red') return lin_regplot(X, y, sgd_lr) plt.show() Intercept: 0.82 Slope: 0.24 plt.plot(range(1, sgd_lr.epochs+1), sgd_lr.cost_) plt.xlabel('Epochs') plt.ylabel('Cost') plt.ylim([0, 0.2]) plt.tight_layout() plt.show() Example 6 - Stochastic Gradient Descent with Minibatches import numpy as np import matplotlib.pyplot as plt from mlxtend.regressor import LinearRegression X = np.array([ 1.0, 2.1, 3.6, 4.2, 6])[:, np.newaxis] y = np.array([ 1.0, 2.0, 3.0, 4.0, 5.0]) sgd_lr = LinearRegression(method='sgd', eta=0.01, epochs=100, random_seed=0, minibatches=3) sgd_lr.fit(X, y) print('Intercept: %.2f' % sgd_lr.b_) print('Slope: %.2f' % sgd_lr.w_) def lin_regplot(X, y, model): plt.scatter(X, y, c='blue') plt.plot(X, model.predict(X), color='red') return lin_regplot(X, y, sgd_lr) plt.show() Intercept: 0.24 Slope: 0.82 plt.plot(range(1, sgd_lr.epochs+1), sgd_lr.cost_) plt.xlabel('Epochs') plt.ylabel('Cost') plt.ylim([0, 0.2]) plt.tight_layout() plt.show() API LinearRegression(method='direct', eta=0.01, epochs=50, minibatches=None, random_seed=None, print_progress=0) Ordinary least squares linear regression. Parameters method : string (default: 'direct') For gradient descent-based optimization, use sgd (see minibatch parameter for further options). Otherwise, if direct (default), the analytical method is used. For alternative, numerically more stable solutions, use either qr (QR decomopisition) or svd (Singular Value Decomposition). eta : float (default: 0.01) solver learning rate (between 0.0 and 1.0). Used with method = 'sgd' . (See methods parameter for details) epochs : int (default: 50) Passes over the training dataset. Prior to each epoch, the dataset is shuffled if minibatches > 1 to prevent cycles in stochastic gradient descent. Used with method = 'sgd' . (See methods parameter for details) minibatches : int (default: None) The number of minibatches for gradient-based optimization. If None: Direct method, QR, or SVD method (see method parameter for details) If 1: Gradient Descent learning If len(y): Stochastic Gradient Descent learning If 1 < minibatches < len(y): Minibatch learning random_seed : int (default: None) Set random state for shuffling and initializing the weights. Used in method = 'sgd' . (See methods parameter for details) print_progress : int (default: 0) Prints progress in fitting to stderr if method = 'sgd' . 0: No output 1: Epochs elapsed and cost 2: 1 plus time elapsed 3: 2 plus estimated time until completion Attributes w_ : 2d-array, shape={n_features, 1} Model weights after fitting. b_ : 1d-array, shape={1,} Bias unit after fitting. cost_ : list Sum of squared errors after each epoch; ignored if solver='normal equation' Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/regressor/LinearRegression/ Methods fit(X, y, init_params=True) Learn model from training data. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values. init_params : bool (default: True) Re-initializes model parameters prior to fitting. Set False to continue training with weights from a previous model fitting. Returns self : object get_params(deep=True) Get parameters for this estimator. Parameters deep : boolean, optional If True, will return the parameters for this estimator and contained subobjects that are estimators. Returns params : mapping of string to any Parameter names mapped to their values.' adapted from https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/base.py Author: Gael Varoquaux gael.varoquaux@normalesup.org License: BSD 3 clause predict(X) Predict targets from X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns target_values : array-like, shape = [n_samples] Predicted target values. set_params( params) Set the parameters of this estimator. The method works on simple estimators as well as on nested objects (such as pipelines). The latter have parameters of the form <component>__<parameter> so that it's possible to update each component of a nested object. Returns self adapted from https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/base.py Author: Gael Varoquaux gael.varoquaux@normalesup.org License: BSD 3 clause ython","title":"LinearRegression: An implementation of ordinary least-squares linear regression"},{"location":"user_guide/regressor/LinearRegression/#linearregression-an-implementation-of-ordinary-least-squares-linear-regression","text":"A implementation of Ordinary Least Squares simple and multiple linear regression. from mlxtend.regressor import LinearRegression","title":"LinearRegression: An implementation of ordinary least-squares linear regression"},{"location":"user_guide/regressor/LinearRegression/#overview","text":"Illustration of a simple linear regression model: In Ordinary Least Squares (OLS) Linear Regression, our goal is to find the line (or hyperplane) that minimizes the vertical offsets. Or in other words, we define the best-fitting line as the line that minimizes the sum of squared errors (SSE) or mean squared error (MSE) between our target variable (y) and our predicted output over all samples i in our dataset of size n . SSE = \\sum_i \\big(\\text{target}^{(i)} - \\text{output}^{(i)}\\big)^2 MSE = \\frac{1}{n} \\times SSE Now, LinearRegression implements a linear regression model for performing ordinary least squares regression using one of the following five approaches: Normal Equations QR Decomposition Method SVD (Singular Value Decomposition) method Gradient Descent Stochastic Gradient Descent","title":"Overview"},{"location":"user_guide/regressor/LinearRegression/#normal-equations-closed-form-solution","text":"The closed-form solution should be preferred for \"smaller\" datasets where calculating (a \"costly\") matrix inverse is not a concern. For very large datasets, or datasets where the inverse of [X^T X] may not exist (the matrix is non-invertible or singular, e.g., in case of perfect multicollinearity), the QR, SVD or gradient descent approaches are to be preferred. The linear function (linear regression model) is defined as: y = w_0x_0 + w_1x_1 + ... + w_mx_m = \\sum_{i=0}^{m} = \\mathbf{w}^T\\mathbf{x} where y is the response variable, \\mathbf{x} is an m -dimensional sample vector, and \\mathbf{w} is the weight vector (vector of coefficients). Note that w_0 represents the y-axis intercept of the model and therefore x_0=1 . Using the closed-form solution (normal equation), we compute the weights of the model as follows: \\mathbf{w} = (\\mathbf{X}^T\\mathbf{X})^{-1}\\mathbf{X}^Ty","title":"Normal Equations (closed-form solution)"},{"location":"user_guide/regressor/LinearRegression/#stable-ols-via-qr-factorization","text":"The QR decomposition method offers a more numerically stable alternative to the closed-form, analytical solution based on the \"normal equations,\" and it can be used to compute the inverse of large matrices more efficiently. QR decomposition method decomposes given matrix into two matrices for which an inverse can be easily obtained. For instance, a given matrix X \\in \\mathbb{R}^{n \\times m} , the QR decomposition into two matrices is: \\mathbf{X = QR}, where Q \\in \\mathbb{R}^{n \\times m} is an orthonormal matrix, such that Q^\\top Q = QQ^\\top = I . The second matrix R \\in \\mathbf{R}^{m \\times m} is an upper triangular matrix. The weight parameters of the ordinary least squares regression model can then be computed as follows [1]: \\mathbf{w} = \\mathbf{R}^{-1}\\mathbf{Q}^\\top y.","title":"Stable OLS via QR Factorization"},{"location":"user_guide/regressor/LinearRegression/#stable-ols-via-singular-value-decomposition","text":"Another alternative way for obtaining the OLS model weights in a numerically stable fashion is by Singular Value Decomposition (SVD), which is defined as: \\mathbf{X}=\\mathbf{U}\\Sigma \\mathbf{V}^\\top, for a given matrix \\mathbf{X} . Then, it can be shown that the pseudo-inverse of X , X^+ , can be obtained as follows [1]: X^+ = \\mathbf{U} \\mathbf{\\Sigma}^+ V^{\\top}. Note that while \\Sigma is the diagonal matrix consisting of singular values of \\mathbf{X} , \\Sigma^{+} is the diagonal matrix consisting of the reciprocals of the singular values. The model weights can then be computed as follows: \\mathbf{w} = \\mathbf{X}^+ \\mathbf{y}. Please note that this OLS method is computationally most inefficient. However, it is a useful approach when the direct method (normal equations) or QR factorization cannot be applied or the normal equations (via \\mathbf{X}^T \\mathbf{X} ) are ill-conditioned [3].","title":"Stable OLS via Singular Value Decomposition"},{"location":"user_guide/regressor/LinearRegression/#gradient-descent-gd-and-stochastic-gradient-descent-sgd","text":"See Gradient Descent and Stochastic Gradient Descent and Deriving the Gradient Descent Rule for Linear Regression and Adaline for details. Random shuffling is implemented as: for one or more epochs randomly shuffle samples in the training set for training sample i compute gradients and perform weight updates","title":"Gradient Descent (GD)  and Stochastic Gradient Descent (SGD)"},{"location":"user_guide/regressor/LinearRegression/#references","text":"[1] Chapter 3, page 55, Linear Methods for Regression. Trevor Hastie; Robert Tibshirani; Jerome Friedman (2009). The Elements of Statistical Learning: Data Mining, Inference, and Prediction (2nd ed.) . New York: Springer. (ISBN 978\u20130\u2013387\u201384858\u20137) [2] G. Strang, Linear Algebra and Its Applications, 2nd Ed., Orlando, FL, Academic Press, Inc., 1980, pp. 139-142. [3] Douglas Wilhelm Harder. Numerical Analysis for Engineering. Section 4.8, ill-conditioned Matrices","title":"References"},{"location":"user_guide/regressor/LinearRegression/#example-1-closed-form-solution","text":"import numpy as np import matplotlib.pyplot as plt from mlxtend.regressor import LinearRegression X = np.array([ 1.0, 2.1, 3.6, 4.2, 6])[:, np.newaxis] y = np.array([ 1.0, 2.0, 3.0, 4.0, 5.0]) ne_lr = LinearRegression() ne_lr.fit(X, y) print('Intercept: %.2f' % ne_lr.b_) print('Slope: %.2f' % ne_lr.w_[0]) def lin_regplot(X, y, model): plt.scatter(X, y, c='blue') plt.plot(X, model.predict(X), color='red') return lin_regplot(X, y, ne_lr) plt.show() Intercept: 0.25 Slope: 0.81","title":"Example 1 - Closed Form Solution"},{"location":"user_guide/regressor/LinearRegression/#example-2-qr-decomposition-method","text":"import numpy as np import matplotlib.pyplot as plt from mlxtend.regressor import LinearRegression X = np.array([ 1.0, 2.1, 3.6, 4.2, 6])[:, np.newaxis] y = np.array([ 1.0, 2.0, 3.0, 4.0, 5.0]) qr_lr = LinearRegression(method='qr') qr_lr.fit(X, y) print('Intercept: %.2f' % qr_lr.b_) print('Slope: %.2f' % qr_lr.w_[0]) def lin_regplot(X, y, model): plt.scatter(X, y, c='blue') plt.plot(X, model.predict(X), color='red') return lin_regplot(X, y, qr_lr) plt.show() Intercept: 0.25 Slope: 0.81","title":"Example 2 - QR decomposition method"},{"location":"user_guide/regressor/LinearRegression/#example-3-svd-method","text":"import numpy as np import matplotlib.pyplot as plt from mlxtend.regressor import LinearRegression X = np.array([ 1.0, 2.1, 3.6, 4.2, 6])[:, np.newaxis] y = np.array([ 1.0, 2.0, 3.0, 4.0, 5.0]) svd_lr = LinearRegression(method='svd') svd_lr.fit(X, y) print('Intercept: %.2f' %svd_lr.b_) print('Slope: %.2f' % svd_lr.w_[0]) def lin_regplot(X, y, model): plt.scatter(X, y, c='blue') plt.plot(X, model.predict(X), color='red') return lin_regplot(X, y, svd_lr) plt.show() Intercept: 0.25 Slope: 0.81","title":"Example 3 - SVD method"},{"location":"user_guide/regressor/LinearRegression/#example-4-gradient-descent","text":"import numpy as np import matplotlib.pyplot as plt from mlxtend.regressor import LinearRegression X = np.array([ 1.0, 2.1, 3.6, 4.2, 6])[:, np.newaxis] y = np.array([ 1.0, 2.0, 3.0, 4.0, 5.0]) gd_lr = LinearRegression(method='sgd', eta=0.005, epochs=100, minibatches=1, random_seed=123, print_progress=3) gd_lr.fit(X, y) print('Intercept: %.2f' % gd_lr.b_) print('Slope: %.2f' % gd_lr.w_) def lin_regplot(X, y, model): plt.scatter(X, y, c='blue') plt.plot(X, model.predict(X), color='red') return lin_regplot(X, y, gd_lr) plt.show() Iteration: 100/100 | Cost 0.08 | Elapsed: 0:00:00 | ETA: 0:00:00 Intercept: 0.22 Slope: 0.82 # Visualizing the cost to check for convergence and plotting the linear model: plt.plot(range(1, gd_lr.epochs+1), gd_lr.cost_) plt.xlabel('Epochs') plt.ylabel('Cost') plt.ylim([0, 0.2]) plt.tight_layout() plt.show()","title":"Example 4 - Gradient Descent"},{"location":"user_guide/regressor/LinearRegression/#example-5-stochastic-gradient-descent","text":"import numpy as np import matplotlib.pyplot as plt from mlxtend.regressor import LinearRegression X = np.array([ 1.0, 2.1, 3.6, 4.2, 6])[:, np.newaxis] y = np.array([ 1.0, 2.0, 3.0, 4.0, 5.0]) sgd_lr = LinearRegression(method='sgd', eta=0.01, epochs=100, random_seed=0, minibatches=len(y)) sgd_lr.fit(X, y) print('Intercept: %.2f' % sgd_lr.w_) print('Slope: %.2f' % sgd_lr.b_) def lin_regplot(X, y, model): plt.scatter(X, y, c='blue') plt.plot(X, model.predict(X), color='red') return lin_regplot(X, y, sgd_lr) plt.show() Intercept: 0.82 Slope: 0.24 plt.plot(range(1, sgd_lr.epochs+1), sgd_lr.cost_) plt.xlabel('Epochs') plt.ylabel('Cost') plt.ylim([0, 0.2]) plt.tight_layout() plt.show()","title":"Example 5 - Stochastic Gradient Descent"},{"location":"user_guide/regressor/LinearRegression/#example-6-stochastic-gradient-descent-with-minibatches","text":"import numpy as np import matplotlib.pyplot as plt from mlxtend.regressor import LinearRegression X = np.array([ 1.0, 2.1, 3.6, 4.2, 6])[:, np.newaxis] y = np.array([ 1.0, 2.0, 3.0, 4.0, 5.0]) sgd_lr = LinearRegression(method='sgd', eta=0.01, epochs=100, random_seed=0, minibatches=3) sgd_lr.fit(X, y) print('Intercept: %.2f' % sgd_lr.b_) print('Slope: %.2f' % sgd_lr.w_) def lin_regplot(X, y, model): plt.scatter(X, y, c='blue') plt.plot(X, model.predict(X), color='red') return lin_regplot(X, y, sgd_lr) plt.show() Intercept: 0.24 Slope: 0.82 plt.plot(range(1, sgd_lr.epochs+1), sgd_lr.cost_) plt.xlabel('Epochs') plt.ylabel('Cost') plt.ylim([0, 0.2]) plt.tight_layout() plt.show()","title":"Example 6 - Stochastic Gradient Descent with Minibatches"},{"location":"user_guide/regressor/LinearRegression/#api","text":"LinearRegression(method='direct', eta=0.01, epochs=50, minibatches=None, random_seed=None, print_progress=0) Ordinary least squares linear regression. Parameters method : string (default: 'direct') For gradient descent-based optimization, use sgd (see minibatch parameter for further options). Otherwise, if direct (default), the analytical method is used. For alternative, numerically more stable solutions, use either qr (QR decomopisition) or svd (Singular Value Decomposition). eta : float (default: 0.01) solver learning rate (between 0.0 and 1.0). Used with method = 'sgd' . (See methods parameter for details) epochs : int (default: 50) Passes over the training dataset. Prior to each epoch, the dataset is shuffled if minibatches > 1 to prevent cycles in stochastic gradient descent. Used with method = 'sgd' . (See methods parameter for details) minibatches : int (default: None) The number of minibatches for gradient-based optimization. If None: Direct method, QR, or SVD method (see method parameter for details) If 1: Gradient Descent learning If len(y): Stochastic Gradient Descent learning If 1 < minibatches < len(y): Minibatch learning random_seed : int (default: None) Set random state for shuffling and initializing the weights. Used in method = 'sgd' . (See methods parameter for details) print_progress : int (default: 0) Prints progress in fitting to stderr if method = 'sgd' . 0: No output 1: Epochs elapsed and cost 2: 1 plus time elapsed 3: 2 plus estimated time until completion Attributes w_ : 2d-array, shape={n_features, 1} Model weights after fitting. b_ : 1d-array, shape={1,} Bias unit after fitting. cost_ : list Sum of squared errors after each epoch; ignored if solver='normal equation' Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/regressor/LinearRegression/","title":"API"},{"location":"user_guide/regressor/LinearRegression/#methods","text":"fit(X, y, init_params=True) Learn model from training data. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape = [n_samples] Target values. init_params : bool (default: True) Re-initializes model parameters prior to fitting. Set False to continue training with weights from a previous model fitting. Returns self : object get_params(deep=True) Get parameters for this estimator. Parameters deep : boolean, optional If True, will return the parameters for this estimator and contained subobjects that are estimators. Returns params : mapping of string to any Parameter names mapped to their values.' adapted from https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/base.py Author: Gael Varoquaux gael.varoquaux@normalesup.org License: BSD 3 clause predict(X) Predict targets from X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns target_values : array-like, shape = [n_samples] Predicted target values. set_params( params) Set the parameters of this estimator. The method works on simple estimators as well as on nested objects (such as pipelines). The latter have parameters of the form <component>__<parameter> so that it's possible to update each component of a nested object. Returns self adapted from https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/base.py Author: Gael Varoquaux gael.varoquaux@normalesup.org License: BSD 3 clause ython","title":"Methods"},{"location":"user_guide/regressor/StackingCVRegressor/","text":"StackingCVRegressor: stacking with cross-validation for regression An ensemble-learning meta-regressor for stacking regression from mlxtend.regressor import StackingCVRegressor Overview Stacking is an ensemble learning technique to combine multiple regression models via a meta-regressor. The StackingCVRegressor extends the standard stacking algorithm (implemented as StackingRegressor ) using out-of-fold predictions to prepare the input data for the level-2 regressor. In the standard stacking procedure, the first-level regressors are fit to the same training set that is used prepare the inputs for the second-level regressor, which may lead to overfitting. The StackingCVRegressor , however, uses the concept of out-of-fold predictions: the dataset is split into k folds, and in k successive rounds, k-1 folds are used to fit the first level regressor. In each round, the first-level regressors are then applied to the remaining 1 subset that was not used for model fitting in each iteration. The resulting predictions are then stacked and provided -- as input data -- to the second-level regressor. After the training of the StackingCVRegressor , the first-level regressors are fit to the entire dataset for optimal predicitons. References Breiman, Leo. \" Stacked regressions. \" Machine learning 24.1 (1996): 49-64. Analogous implementation: StackingCVClassifier Example 1: Boston Housing Data Predictions In this example we evaluate some basic prediction models on the boston housing dataset and see how the R^2 and MSE scores are affected by combining the models with StackingCVRegressor . The code output below demonstrates that the stacked model performs the best on this dataset -- slightly better than the best single regression model. from mlxtend.regressor import StackingCVRegressor from sklearn.datasets import load_boston from sklearn.svm import SVR from sklearn.linear_model import Lasso from sklearn.ensemble import RandomForestRegressor from sklearn.model_selection import cross_val_score import numpy as np RANDOM_SEED = 42 X, y = load_boston(return_X_y=True) svr = SVR(kernel='linear') lasso = Lasso() rf = RandomForestRegressor(n_estimators=5, random_state=RANDOM_SEED) # Starting from v0.16.0, StackingCVRegressor supports # `random_state` to get deterministic result. stack = StackingCVRegressor(regressors=(svr, lasso, rf), meta_regressor=lasso, random_state=RANDOM_SEED) print('5-fold cross validation scores:\\n') for clf, label in zip([svr, lasso, rf, stack], ['SVM', 'Lasso', 'Random Forest', 'StackingCVRegressor']): scores = cross_val_score(clf, X, y, cv=5) print(\"R^2 Score: %0.2f (+/- %0.2f) [%s]\" % ( scores.mean(), scores.std(), label)) 5-fold cross validation scores: R^2 Score: 0.46 (+/- 0.29) [SVM] R^2 Score: 0.43 (+/- 0.14) [Lasso] R^2 Score: 0.53 (+/- 0.28) [Random Forest] R^2 Score: 0.57 (+/- 0.24) [StackingCVRegressor] stack = StackingCVRegressor(regressors=(svr, lasso, rf), meta_regressor=lasso) print('5-fold cross validation scores:\\n') for clf, label in zip([svr, lasso, rf, stack], ['SVM', 'Lasso', 'Random Forest', 'StackingCVRegressor']): scores = cross_val_score(clf, X, y, cv=5, scoring='neg_mean_squared_error') print(\"Neg. MSE Score: %0.2f (+/- %0.2f) [%s]\" % ( scores.mean(), scores.std(), label)) 5-fold cross validation scores: Neg. MSE Score: -33.33 (+/- 22.36) [SVM] Neg. MSE Score: -35.53 (+/- 16.99) [Lasso] Neg. MSE Score: -27.08 (+/- 15.67) [Random Forest] Neg. MSE Score: -25.85 (+/- 17.22) [StackingCVRegressor] Example 2: GridSearchCV with Stacking In this second example we demonstrate how StackingCVRegressor works in combination with GridSearchCV . The stack still allows tuning hyper parameters of the base and meta models! For instance, we can use estimator.get_params().keys() to get a full list of tunable parameters. from mlxtend.regressor import StackingCVRegressor from sklearn.datasets import load_boston from sklearn.linear_model import Lasso from sklearn.linear_model import Ridge from sklearn.ensemble import RandomForestRegressor from sklearn.model_selection import GridSearchCV X, y = load_boston(return_X_y=True) ridge = Ridge(random_state=RANDOM_SEED) lasso = Lasso(random_state=RANDOM_SEED) rf = RandomForestRegressor(random_state=RANDOM_SEED) stack = StackingCVRegressor(regressors=(lasso, ridge), meta_regressor=rf, random_state=RANDOM_SEED, use_features_in_secondary=True) params = {'lasso__alpha': [0.1, 1.0, 10.0], 'ridge__alpha': [0.1, 1.0, 10.0]} grid = GridSearchCV( estimator=stack, param_grid={ 'lasso__alpha': [x/5.0 for x in range(1, 10)], 'ridge__alpha': [x/20.0 for x in range(1, 10)], 'meta_regressor__n_estimators': [10, 100] }, cv=5, refit=True ) grid.fit(X, y) print(\"Best: %f using %s\" % (grid.best_score_, grid.best_params_)) Best: 0.679151 using {'lasso__alpha': 1.6, 'meta_regressor__n_estimators': 10, 'ridge__alpha': 0.4} cv_keys = ('mean_test_score', 'std_test_score', 'params') for r, _ in enumerate(grid.cv_results_['mean_test_score']): print(\"%0.3f +/- %0.2f %r\" % (grid.cv_results_[cv_keys[0]][r], grid.cv_results_[cv_keys[1]][r] / 2.0, grid.cv_results_[cv_keys[2]][r])) if r > 10: break print('...') print('Best parameters: %s' % grid.best_params_) print('Accuracy: %.2f' % grid.best_score_) 0.632 +/- 0.09 {'lasso__alpha': 0.2, 'meta_regressor__n_estimators': 10, 'ridge__alpha': 0.05} 0.645 +/- 0.08 {'lasso__alpha': 0.2, 'meta_regressor__n_estimators': 10, 'ridge__alpha': 0.1} 0.641 +/- 0.08 {'lasso__alpha': 0.2, 'meta_regressor__n_estimators': 10, 'ridge__alpha': 0.15} 0.653 +/- 0.08 {'lasso__alpha': 0.2, 'meta_regressor__n_estimators': 10, 'ridge__alpha': 0.2} 0.622 +/- 0.10 {'lasso__alpha': 0.2, 'meta_regressor__n_estimators': 10, 'ridge__alpha': 0.25} 0.630 +/- 0.09 {'lasso__alpha': 0.2, 'meta_regressor__n_estimators': 10, 'ridge__alpha': 0.3} 0.630 +/- 0.09 {'lasso__alpha': 0.2, 'meta_regressor__n_estimators': 10, 'ridge__alpha': 0.35} 0.642 +/- 0.09 {'lasso__alpha': 0.2, 'meta_regressor__n_estimators': 10, 'ridge__alpha': 0.4} 0.654 +/- 0.08 {'lasso__alpha': 0.2, 'meta_regressor__n_estimators': 10, 'ridge__alpha': 0.45} 0.642 +/- 0.09 {'lasso__alpha': 0.2, 'meta_regressor__n_estimators': 100, 'ridge__alpha': 0.05} 0.645 +/- 0.09 {'lasso__alpha': 0.2, 'meta_regressor__n_estimators': 100, 'ridge__alpha': 0.1} 0.648 +/- 0.09 {'lasso__alpha': 0.2, 'meta_regressor__n_estimators': 100, 'ridge__alpha': 0.15} ... Best parameters: {'lasso__alpha': 1.6, 'meta_regressor__n_estimators': 10, 'ridge__alpha': 0.4} Accuracy: 0.68 Note The StackingCVRegressor also enables grid search over the regressors and even a single base regressor. When there are level-mixed hyperparameters, GridSearchCV will try to replace hyperparameters in a top-down order, i.e., regressors -> single base regressor -> regressor hyperparameter. For instance, given a hyperparameter grid such as params = {'randomforestregressor__n_estimators': [1, 100], 'regressors': [(regr1, regr1, regr1), (regr2, regr3)]} it will first use the instance settings of either (regr1, regr2, regr3) or (regr2, regr3) . Then it will replace the 'n_estimators' settings for a matching regressor based on 'randomforestregressor__n_estimators': [1, 100] . API StackingCVRegressor(regressors, meta_regressor, cv=5, shuffle=True, random_state=None, verbose=0, refit=True, use_features_in_secondary=False, store_train_meta_features=False, n_jobs=None, pre_dispatch='2 n_jobs', multi_output=False)* A 'Stacking Cross-Validation' regressor for scikit-learn estimators. Parameters regressors : array-like, shape = [n_regressors] A list of regressors. Invoking the fit method on the StackingCVRegressor will fit clones of these original regressors that will be stored in the class attribute self.regr_ . meta_regressor : object The meta-regressor to be fitted on the ensemble of regressor cv : int, cross-validation generator or iterable, optional (default: 5) Determines the cross-validation splitting strategy. Possible inputs for cv are: - None, to use the default 5-fold cross validation, - integer, to specify the number of folds in a KFold , - An object to be used as a cross-validation generator. - An iterable yielding train, test splits. For integer/None inputs, it will use KFold cross-validation shuffle : bool (default: True) If True, and the cv argument is integer, the training data will be shuffled at fitting stage prior to cross-validation. If the cv argument is a specific cross validation technique, this argument is omitted. random_state : int, RandomState instance or None, optional (default: None) Constrols the randomness of the cv splitter. Used when cv is integer and shuffle=True . New in v0.16.0. verbose : int, optional (default=0) Controls the verbosity of the building process. New in v0.16.0 refit : bool (default: True) Clones the regressors for stacking regression if True (default) or else uses the original ones, which will be refitted on the dataset upon calling the fit method. Setting refit=False is recommended if you are working with estimators that are supporting the scikit-learn fit/predict API interface but are not compatible to scikit-learn's clone function. use_features_in_secondary : bool (default: False) If True, the meta-regressor will be trained both on the predictions of the original regressors and the original dataset. If False, the meta-regressor will be trained only on the predictions of the original regressors. store_train_meta_features : bool (default: False) If True, the meta-features computed from the training data used for fitting the meta-regressor stored in the self.train_meta_features_ array, which can be accessed after calling fit . n_jobs : int or None, optional (default=None) The number of CPUs to use to do the computation. None means 1 unless in a :obj: joblib.parallel_backend context. -1 means using all processors. See :term: Glossary <n_jobs> for more details. New in v0.16.0. pre_dispatch : int, or string, optional Controls the number of jobs that get dispatched during parallel execution. Reducing this number can be useful to avoid an explosion of memory consumption when more jobs get dispatched than CPUs can process. This parameter can be: - None, in which case all the jobs are immediately created and spawned. Use this for lightweight and fast-running jobs, to avoid delays due to on-demand spawning of the jobs - An int, giving the exact number of total jobs that are spawned - A string, giving an expression as a function of n_jobs, as in '2*n_jobs' multi_output : bool (default: False) If True, allow multi-output targets, but forbid nan or inf values. If False, y will be checked to be a vector. (New in v0.19.0.) Attributes train_meta_features : numpy array, shape = [n_samples, n_regressors] meta-features for training data, where n_samples is the number of samples in training data and len(self.regressors) is the number of regressors. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/regressor/StackingCVRegressor/ Methods fit(X, y, groups=None, sample_weight=None) Fit ensemble regressors and the meta-regressor. Parameters X : numpy array, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : numpy array, shape = [n_samples] or [n_samples, n_targets] Target values. Multiple targets are supported only if self.multi_output is True. groups : numpy array/None, shape = [n_samples] The group that each sample belongs to. This is used by specific folding strategies such as GroupKFold() sample_weight : array-like, shape = [n_samples], optional Sample weights passed as sample_weights to each regressor in the regressors list as well as the meta_regressor. Raises error if some regressor does not support sample_weight in the fit() method. Returns self : object fit_transform(X, y=None, fit_params) Fit to data, then transform it. Fits transformer to `X` and `y` with optional parameters `fit_params` and returns a transformed version of `X`. Parameters X : array-like of shape (n_samples, n_features) Input samples. y : array-like of shape (n_samples,) or (n_samples, n_outputs), default=None Target values (None for unsupervised transformations). **fit_params : dict Additional fit parameters. Returns X_new : ndarray array of shape (n_samples, n_features_new) Transformed array. get_params(deep=True) Get parameters for this estimator. Parameters deep : bool, default=True If True, will return the parameters for this estimator and contained subobjects that are estimators. Returns params : dict Parameter names mapped to their values. predict(X) Predict target values for X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns y_target : array-like, shape = [n_samples] or [n_samples, n_targets] Predicted target values. predict_meta_features(X) Get meta-features of test-data. Parameters X : numpy array, shape = [n_samples, n_features] Test vectors, where n_samples is the number of samples and n_features is the number of features. Returns meta-features : numpy array, shape = [n_samples, len(self.regressors)] meta-features for test data, where n_samples is the number of samples in test data and len(self.regressors) is the number of regressors. If self.multi_output is True, then the number of columns is len(self.regressors) * n_targets. score(X, y, sample_weight=None) Return the coefficient of determination :math: R^2 of the prediction. The coefficient :math:`R^2` is defined as :math:`(1 - \\frac{u}{v})`, where :math:`u` is the residual sum of squares ``((y_true - y_pred) ** 2).sum() and :math:`v` is the total sum of squares ((y_true - y_true.mean()) ** 2).sum()``. The best possible score is 1.0 and it can be negative (because the model can be arbitrarily worse). A constant model that always predicts the expected value of y , disregarding the input features, would get a :math: R^2 score of 0.0. Parameters X : array-like of shape (n_samples, n_features) Test samples. For some estimators this may be a precomputed kernel matrix or a list of generic objects instead with shape (n_samples, n_samples_fitted) , where n_samples_fitted is the number of samples used in the fitting for the estimator. y : array-like of shape (n_samples,) or (n_samples, n_outputs) True values for X . sample_weight : array-like of shape (n_samples,), default=None Sample weights. Returns score : float :math: R^2 of self.predict(X) wrt. y . Notes The :math: R^2 score used when calling score on a regressor uses multioutput='uniform_average' from version 0.23 to keep consistent with default value of :func: ~sklearn.metrics.r2_score . This influences the score method of all the multioutput regressors (except for :class: ~sklearn.multioutput.MultiOutputRegressor ). set_params( params) Set the parameters of this estimator. Valid parameter keys can be listed with ``get_params()``. Returns self Properties named_regressors Returns List of named estimator tuples, like [('svc', SVC(...))]","title":"StackingCVRegressor: stacking with cross-validation for regression"},{"location":"user_guide/regressor/StackingCVRegressor/#stackingcvregressor-stacking-with-cross-validation-for-regression","text":"An ensemble-learning meta-regressor for stacking regression from mlxtend.regressor import StackingCVRegressor","title":"StackingCVRegressor: stacking with cross-validation for regression"},{"location":"user_guide/regressor/StackingCVRegressor/#overview","text":"Stacking is an ensemble learning technique to combine multiple regression models via a meta-regressor. The StackingCVRegressor extends the standard stacking algorithm (implemented as StackingRegressor ) using out-of-fold predictions to prepare the input data for the level-2 regressor. In the standard stacking procedure, the first-level regressors are fit to the same training set that is used prepare the inputs for the second-level regressor, which may lead to overfitting. The StackingCVRegressor , however, uses the concept of out-of-fold predictions: the dataset is split into k folds, and in k successive rounds, k-1 folds are used to fit the first level regressor. In each round, the first-level regressors are then applied to the remaining 1 subset that was not used for model fitting in each iteration. The resulting predictions are then stacked and provided -- as input data -- to the second-level regressor. After the training of the StackingCVRegressor , the first-level regressors are fit to the entire dataset for optimal predicitons.","title":"Overview"},{"location":"user_guide/regressor/StackingCVRegressor/#references","text":"Breiman, Leo. \" Stacked regressions. \" Machine learning 24.1 (1996): 49-64. Analogous implementation: StackingCVClassifier","title":"References"},{"location":"user_guide/regressor/StackingCVRegressor/#example-1-boston-housing-data-predictions","text":"In this example we evaluate some basic prediction models on the boston housing dataset and see how the R^2 and MSE scores are affected by combining the models with StackingCVRegressor . The code output below demonstrates that the stacked model performs the best on this dataset -- slightly better than the best single regression model. from mlxtend.regressor import StackingCVRegressor from sklearn.datasets import load_boston from sklearn.svm import SVR from sklearn.linear_model import Lasso from sklearn.ensemble import RandomForestRegressor from sklearn.model_selection import cross_val_score import numpy as np RANDOM_SEED = 42 X, y = load_boston(return_X_y=True) svr = SVR(kernel='linear') lasso = Lasso() rf = RandomForestRegressor(n_estimators=5, random_state=RANDOM_SEED) # Starting from v0.16.0, StackingCVRegressor supports # `random_state` to get deterministic result. stack = StackingCVRegressor(regressors=(svr, lasso, rf), meta_regressor=lasso, random_state=RANDOM_SEED) print('5-fold cross validation scores:\\n') for clf, label in zip([svr, lasso, rf, stack], ['SVM', 'Lasso', 'Random Forest', 'StackingCVRegressor']): scores = cross_val_score(clf, X, y, cv=5) print(\"R^2 Score: %0.2f (+/- %0.2f) [%s]\" % ( scores.mean(), scores.std(), label)) 5-fold cross validation scores: R^2 Score: 0.46 (+/- 0.29) [SVM] R^2 Score: 0.43 (+/- 0.14) [Lasso] R^2 Score: 0.53 (+/- 0.28) [Random Forest] R^2 Score: 0.57 (+/- 0.24) [StackingCVRegressor] stack = StackingCVRegressor(regressors=(svr, lasso, rf), meta_regressor=lasso) print('5-fold cross validation scores:\\n') for clf, label in zip([svr, lasso, rf, stack], ['SVM', 'Lasso', 'Random Forest', 'StackingCVRegressor']): scores = cross_val_score(clf, X, y, cv=5, scoring='neg_mean_squared_error') print(\"Neg. MSE Score: %0.2f (+/- %0.2f) [%s]\" % ( scores.mean(), scores.std(), label)) 5-fold cross validation scores: Neg. MSE Score: -33.33 (+/- 22.36) [SVM] Neg. MSE Score: -35.53 (+/- 16.99) [Lasso] Neg. MSE Score: -27.08 (+/- 15.67) [Random Forest] Neg. MSE Score: -25.85 (+/- 17.22) [StackingCVRegressor]","title":"Example 1: Boston Housing Data Predictions"},{"location":"user_guide/regressor/StackingCVRegressor/#example-2-gridsearchcv-with-stacking","text":"In this second example we demonstrate how StackingCVRegressor works in combination with GridSearchCV . The stack still allows tuning hyper parameters of the base and meta models! For instance, we can use estimator.get_params().keys() to get a full list of tunable parameters. from mlxtend.regressor import StackingCVRegressor from sklearn.datasets import load_boston from sklearn.linear_model import Lasso from sklearn.linear_model import Ridge from sklearn.ensemble import RandomForestRegressor from sklearn.model_selection import GridSearchCV X, y = load_boston(return_X_y=True) ridge = Ridge(random_state=RANDOM_SEED) lasso = Lasso(random_state=RANDOM_SEED) rf = RandomForestRegressor(random_state=RANDOM_SEED) stack = StackingCVRegressor(regressors=(lasso, ridge), meta_regressor=rf, random_state=RANDOM_SEED, use_features_in_secondary=True) params = {'lasso__alpha': [0.1, 1.0, 10.0], 'ridge__alpha': [0.1, 1.0, 10.0]} grid = GridSearchCV( estimator=stack, param_grid={ 'lasso__alpha': [x/5.0 for x in range(1, 10)], 'ridge__alpha': [x/20.0 for x in range(1, 10)], 'meta_regressor__n_estimators': [10, 100] }, cv=5, refit=True ) grid.fit(X, y) print(\"Best: %f using %s\" % (grid.best_score_, grid.best_params_)) Best: 0.679151 using {'lasso__alpha': 1.6, 'meta_regressor__n_estimators': 10, 'ridge__alpha': 0.4} cv_keys = ('mean_test_score', 'std_test_score', 'params') for r, _ in enumerate(grid.cv_results_['mean_test_score']): print(\"%0.3f +/- %0.2f %r\" % (grid.cv_results_[cv_keys[0]][r], grid.cv_results_[cv_keys[1]][r] / 2.0, grid.cv_results_[cv_keys[2]][r])) if r > 10: break print('...') print('Best parameters: %s' % grid.best_params_) print('Accuracy: %.2f' % grid.best_score_) 0.632 +/- 0.09 {'lasso__alpha': 0.2, 'meta_regressor__n_estimators': 10, 'ridge__alpha': 0.05} 0.645 +/- 0.08 {'lasso__alpha': 0.2, 'meta_regressor__n_estimators': 10, 'ridge__alpha': 0.1} 0.641 +/- 0.08 {'lasso__alpha': 0.2, 'meta_regressor__n_estimators': 10, 'ridge__alpha': 0.15} 0.653 +/- 0.08 {'lasso__alpha': 0.2, 'meta_regressor__n_estimators': 10, 'ridge__alpha': 0.2} 0.622 +/- 0.10 {'lasso__alpha': 0.2, 'meta_regressor__n_estimators': 10, 'ridge__alpha': 0.25} 0.630 +/- 0.09 {'lasso__alpha': 0.2, 'meta_regressor__n_estimators': 10, 'ridge__alpha': 0.3} 0.630 +/- 0.09 {'lasso__alpha': 0.2, 'meta_regressor__n_estimators': 10, 'ridge__alpha': 0.35} 0.642 +/- 0.09 {'lasso__alpha': 0.2, 'meta_regressor__n_estimators': 10, 'ridge__alpha': 0.4} 0.654 +/- 0.08 {'lasso__alpha': 0.2, 'meta_regressor__n_estimators': 10, 'ridge__alpha': 0.45} 0.642 +/- 0.09 {'lasso__alpha': 0.2, 'meta_regressor__n_estimators': 100, 'ridge__alpha': 0.05} 0.645 +/- 0.09 {'lasso__alpha': 0.2, 'meta_regressor__n_estimators': 100, 'ridge__alpha': 0.1} 0.648 +/- 0.09 {'lasso__alpha': 0.2, 'meta_regressor__n_estimators': 100, 'ridge__alpha': 0.15} ... Best parameters: {'lasso__alpha': 1.6, 'meta_regressor__n_estimators': 10, 'ridge__alpha': 0.4} Accuracy: 0.68 Note The StackingCVRegressor also enables grid search over the regressors and even a single base regressor. When there are level-mixed hyperparameters, GridSearchCV will try to replace hyperparameters in a top-down order, i.e., regressors -> single base regressor -> regressor hyperparameter. For instance, given a hyperparameter grid such as params = {'randomforestregressor__n_estimators': [1, 100], 'regressors': [(regr1, regr1, regr1), (regr2, regr3)]} it will first use the instance settings of either (regr1, regr2, regr3) or (regr2, regr3) . Then it will replace the 'n_estimators' settings for a matching regressor based on 'randomforestregressor__n_estimators': [1, 100] .","title":"Example 2: GridSearchCV with Stacking"},{"location":"user_guide/regressor/StackingCVRegressor/#api","text":"StackingCVRegressor(regressors, meta_regressor, cv=5, shuffle=True, random_state=None, verbose=0, refit=True, use_features_in_secondary=False, store_train_meta_features=False, n_jobs=None, pre_dispatch='2 n_jobs', multi_output=False)* A 'Stacking Cross-Validation' regressor for scikit-learn estimators. Parameters regressors : array-like, shape = [n_regressors] A list of regressors. Invoking the fit method on the StackingCVRegressor will fit clones of these original regressors that will be stored in the class attribute self.regr_ . meta_regressor : object The meta-regressor to be fitted on the ensemble of regressor cv : int, cross-validation generator or iterable, optional (default: 5) Determines the cross-validation splitting strategy. Possible inputs for cv are: - None, to use the default 5-fold cross validation, - integer, to specify the number of folds in a KFold , - An object to be used as a cross-validation generator. - An iterable yielding train, test splits. For integer/None inputs, it will use KFold cross-validation shuffle : bool (default: True) If True, and the cv argument is integer, the training data will be shuffled at fitting stage prior to cross-validation. If the cv argument is a specific cross validation technique, this argument is omitted. random_state : int, RandomState instance or None, optional (default: None) Constrols the randomness of the cv splitter. Used when cv is integer and shuffle=True . New in v0.16.0. verbose : int, optional (default=0) Controls the verbosity of the building process. New in v0.16.0 refit : bool (default: True) Clones the regressors for stacking regression if True (default) or else uses the original ones, which will be refitted on the dataset upon calling the fit method. Setting refit=False is recommended if you are working with estimators that are supporting the scikit-learn fit/predict API interface but are not compatible to scikit-learn's clone function. use_features_in_secondary : bool (default: False) If True, the meta-regressor will be trained both on the predictions of the original regressors and the original dataset. If False, the meta-regressor will be trained only on the predictions of the original regressors. store_train_meta_features : bool (default: False) If True, the meta-features computed from the training data used for fitting the meta-regressor stored in the self.train_meta_features_ array, which can be accessed after calling fit . n_jobs : int or None, optional (default=None) The number of CPUs to use to do the computation. None means 1 unless in a :obj: joblib.parallel_backend context. -1 means using all processors. See :term: Glossary <n_jobs> for more details. New in v0.16.0. pre_dispatch : int, or string, optional Controls the number of jobs that get dispatched during parallel execution. Reducing this number can be useful to avoid an explosion of memory consumption when more jobs get dispatched than CPUs can process. This parameter can be: - None, in which case all the jobs are immediately created and spawned. Use this for lightweight and fast-running jobs, to avoid delays due to on-demand spawning of the jobs - An int, giving the exact number of total jobs that are spawned - A string, giving an expression as a function of n_jobs, as in '2*n_jobs' multi_output : bool (default: False) If True, allow multi-output targets, but forbid nan or inf values. If False, y will be checked to be a vector. (New in v0.19.0.) Attributes train_meta_features : numpy array, shape = [n_samples, n_regressors] meta-features for training data, where n_samples is the number of samples in training data and len(self.regressors) is the number of regressors. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/regressor/StackingCVRegressor/","title":"API"},{"location":"user_guide/regressor/StackingCVRegressor/#methods","text":"fit(X, y, groups=None, sample_weight=None) Fit ensemble regressors and the meta-regressor. Parameters X : numpy array, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : numpy array, shape = [n_samples] or [n_samples, n_targets] Target values. Multiple targets are supported only if self.multi_output is True. groups : numpy array/None, shape = [n_samples] The group that each sample belongs to. This is used by specific folding strategies such as GroupKFold() sample_weight : array-like, shape = [n_samples], optional Sample weights passed as sample_weights to each regressor in the regressors list as well as the meta_regressor. Raises error if some regressor does not support sample_weight in the fit() method. Returns self : object fit_transform(X, y=None, fit_params) Fit to data, then transform it. Fits transformer to `X` and `y` with optional parameters `fit_params` and returns a transformed version of `X`. Parameters X : array-like of shape (n_samples, n_features) Input samples. y : array-like of shape (n_samples,) or (n_samples, n_outputs), default=None Target values (None for unsupervised transformations). **fit_params : dict Additional fit parameters. Returns X_new : ndarray array of shape (n_samples, n_features_new) Transformed array. get_params(deep=True) Get parameters for this estimator. Parameters deep : bool, default=True If True, will return the parameters for this estimator and contained subobjects that are estimators. Returns params : dict Parameter names mapped to their values. predict(X) Predict target values for X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns y_target : array-like, shape = [n_samples] or [n_samples, n_targets] Predicted target values. predict_meta_features(X) Get meta-features of test-data. Parameters X : numpy array, shape = [n_samples, n_features] Test vectors, where n_samples is the number of samples and n_features is the number of features. Returns meta-features : numpy array, shape = [n_samples, len(self.regressors)] meta-features for test data, where n_samples is the number of samples in test data and len(self.regressors) is the number of regressors. If self.multi_output is True, then the number of columns is len(self.regressors) * n_targets. score(X, y, sample_weight=None) Return the coefficient of determination :math: R^2 of the prediction. The coefficient :math:`R^2` is defined as :math:`(1 - \\frac{u}{v})`, where :math:`u` is the residual sum of squares ``((y_true - y_pred) ** 2).sum() and :math:`v` is the total sum of squares ((y_true - y_true.mean()) ** 2).sum()``. The best possible score is 1.0 and it can be negative (because the model can be arbitrarily worse). A constant model that always predicts the expected value of y , disregarding the input features, would get a :math: R^2 score of 0.0. Parameters X : array-like of shape (n_samples, n_features) Test samples. For some estimators this may be a precomputed kernel matrix or a list of generic objects instead with shape (n_samples, n_samples_fitted) , where n_samples_fitted is the number of samples used in the fitting for the estimator. y : array-like of shape (n_samples,) or (n_samples, n_outputs) True values for X . sample_weight : array-like of shape (n_samples,), default=None Sample weights. Returns score : float :math: R^2 of self.predict(X) wrt. y . Notes The :math: R^2 score used when calling score on a regressor uses multioutput='uniform_average' from version 0.23 to keep consistent with default value of :func: ~sklearn.metrics.r2_score . This influences the score method of all the multioutput regressors (except for :class: ~sklearn.multioutput.MultiOutputRegressor ). set_params( params) Set the parameters of this estimator. Valid parameter keys can be listed with ``get_params()``. Returns self","title":"Methods"},{"location":"user_guide/regressor/StackingCVRegressor/#properties","text":"named_regressors Returns List of named estimator tuples, like [('svc', SVC(...))]","title":"Properties"},{"location":"user_guide/regressor/StackingRegressor/","text":"StackingRegressor: a simple stacking implementation for regression An ensemble-learning meta-regressor for stacking regression from mlxtend.regressor import StackingRegressor Overview Stacking regression is an ensemble learning technique to combine multiple regression models via a meta-regressor. The individual regression models are trained based on the complete training set; then, the meta-regressor is fitted based on the outputs -- meta-features -- of the individual regression models in the ensemble. References Breiman, Leo. \" Stacked regressions. \" Machine learning 24.1 (1996): 49-64. Example 1 - Simple Stacked Regression from mlxtend.regressor import StackingRegressor from mlxtend.data import boston_housing_data from sklearn.linear_model import LinearRegression from sklearn.linear_model import Ridge from sklearn.svm import SVR import matplotlib.pyplot as plt import numpy as np import warnings warnings.simplefilter('ignore') # Generating a sample dataset np.random.seed(1) X = np.sort(5 * np.random.rand(40, 1), axis=0) y = np.sin(X).ravel() y[::5] += 3 * (0.5 - np.random.rand(8)) # Initializing models lr = LinearRegression() svr_lin = SVR(kernel='linear') ridge = Ridge(random_state=1) svr_rbf = SVR(kernel='rbf') stregr = StackingRegressor(regressors=[svr_lin, lr, ridge], meta_regressor=svr_rbf) # Training the stacking classifier stregr.fit(X, y) stregr.predict(X) # Evaluate and visualize the fit print(\"Mean Squared Error: %.4f\" % np.mean((stregr.predict(X) - y) ** 2)) print('Variance Score: %.4f' % stregr.score(X, y)) with plt.style.context(('seaborn-whitegrid')): plt.scatter(X, y, c='lightgray') plt.plot(X, stregr.predict(X), c='darkgreen', lw=2) plt.show() Mean Squared Error: 0.1846 Variance Score: 0.7329 stregr StackingRegressor(meta_regressor=SVR(), regressors=[SVR(kernel='linear'), LinearRegression(), Ridge(random_state=1)]) Example 2 - Stacked Regression and GridSearch In this second example we demonstrate how StackingCVRegressor works in combination with GridSearchCV . The stack still allows tuning hyper parameters of the base and meta models! For instance, we can use estimator.get_params().keys() to get a full list of tunable parameters. from sklearn.model_selection import GridSearchCV from sklearn.linear_model import Lasso # Initializing models lr = LinearRegression() svr_lin = SVR(kernel='linear') ridge = Ridge(random_state=1) lasso = Lasso(random_state=1) svr_rbf = SVR(kernel='rbf') regressors = [svr_lin, lr, ridge, lasso] stregr = StackingRegressor(regressors=regressors, meta_regressor=svr_rbf) params = {'lasso__alpha': [0.1, 1.0, 10.0], 'ridge__alpha': [0.1, 1.0, 10.0], 'svr__C': [0.1, 1.0, 10.0], 'meta_regressor__C': [0.1, 1.0, 10.0, 100.0], 'meta_regressor__gamma': [0.1, 1.0, 10.0]} grid = GridSearchCV(estimator=stregr, param_grid=params, cv=5, refit=True) grid.fit(X, y) print(\"Best: %f using %s\" % (grid.best_score_, grid.best_params_)) Best: -0.082717 using {'lasso__alpha': 0.1, 'meta_regressor__C': 1.0, 'meta_regressor__gamma': 1.0, 'ridge__alpha': 0.1, 'svr__C': 10.0} cv_keys = ('mean_test_score', 'std_test_score', 'params') for r, _ in enumerate(grid.cv_results_['mean_test_score']): print(\"%0.3f +/- %0.2f %r\" % (grid.cv_results_[cv_keys[0]][r], grid.cv_results_[cv_keys[1]][r] / 2.0, grid.cv_results_[cv_keys[2]][r])) if r > 10: break print('...') print('Best parameters: %s' % grid.best_params_) print('Accuracy: %.2f' % grid.best_score_) -9.810 +/- 6.86 {'lasso__alpha': 0.1, 'meta_regressor__C': 0.1, 'meta_regressor__gamma': 0.1, 'ridge__alpha': 0.1, 'svr__C': 0.1} -9.591 +/- 6.67 {'lasso__alpha': 0.1, 'meta_regressor__C': 0.1, 'meta_regressor__gamma': 0.1, 'ridge__alpha': 0.1, 'svr__C': 1.0} -9.591 +/- 6.67 {'lasso__alpha': 0.1, 'meta_regressor__C': 0.1, 'meta_regressor__gamma': 0.1, 'ridge__alpha': 0.1, 'svr__C': 10.0} -9.819 +/- 6.87 {'lasso__alpha': 0.1, 'meta_regressor__C': 0.1, 'meta_regressor__gamma': 0.1, 'ridge__alpha': 1.0, 'svr__C': 0.1} -9.600 +/- 6.68 {'lasso__alpha': 0.1, 'meta_regressor__C': 0.1, 'meta_regressor__gamma': 0.1, 'ridge__alpha': 1.0, 'svr__C': 1.0} -9.600 +/- 6.68 {'lasso__alpha': 0.1, 'meta_regressor__C': 0.1, 'meta_regressor__gamma': 0.1, 'ridge__alpha': 1.0, 'svr__C': 10.0} -9.878 +/- 6.91 {'lasso__alpha': 0.1, 'meta_regressor__C': 0.1, 'meta_regressor__gamma': 0.1, 'ridge__alpha': 10.0, 'svr__C': 0.1} -9.665 +/- 6.71 {'lasso__alpha': 0.1, 'meta_regressor__C': 0.1, 'meta_regressor__gamma': 0.1, 'ridge__alpha': 10.0, 'svr__C': 1.0} -9.665 +/- 6.71 {'lasso__alpha': 0.1, 'meta_regressor__C': 0.1, 'meta_regressor__gamma': 0.1, 'ridge__alpha': 10.0, 'svr__C': 10.0} -4.839 +/- 3.98 {'lasso__alpha': 0.1, 'meta_regressor__C': 0.1, 'meta_regressor__gamma': 1.0, 'ridge__alpha': 0.1, 'svr__C': 0.1} -3.986 +/- 3.16 {'lasso__alpha': 0.1, 'meta_regressor__C': 0.1, 'meta_regressor__gamma': 1.0, 'ridge__alpha': 0.1, 'svr__C': 1.0} -3.986 +/- 3.16 {'lasso__alpha': 0.1, 'meta_regressor__C': 0.1, 'meta_regressor__gamma': 1.0, 'ridge__alpha': 0.1, 'svr__C': 10.0} ... Best parameters: {'lasso__alpha': 0.1, 'meta_regressor__C': 1.0, 'meta_regressor__gamma': 1.0, 'ridge__alpha': 0.1, 'svr__C': 10.0} Accuracy: -0.08 # Evaluate and visualize the fit print(\"Mean Squared Error: %.4f\" % np.mean((grid.predict(X) - y) ** 2)) print('Variance Score: %.4f' % grid.score(X, y)) with plt.style.context(('seaborn-whitegrid')): plt.scatter(X, y, c='lightgray') plt.plot(X, grid.predict(X), c='darkgreen', lw=2) plt.show() Mean Squared Error: 0.1845 Variance Score: 0.7330 Note The StackingCVRegressor also enables grid search over the regressors and even a single base regressor. When there are level-mixed hyperparameters, GridSearchCV will try to replace hyperparameters in a top-down order, i.e., regressors -> single base regressor -> regressor hyperparameter. For instance, given a hyperparameter grid such as params = {'randomforestregressor__n_estimators': [1, 100], 'regressors': [(regr1, regr1, regr1), (regr2, regr3)]} it will first use the instance settings of either (regr1, regr2, regr3) or (regr2, regr3) . Then it will replace the 'n_estimators' settings for a matching regressor based on 'randomforestregressor__n_estimators': [1, 100] . API StackingRegressor(regressors, meta_regressor, verbose=0, use_features_in_secondary=False, store_train_meta_features=False, refit=True, multi_output=False) A Stacking regressor for scikit-learn estimators for regression. Parameters regressors : array-like, shape = [n_regressors] A list of regressors. Invoking the fit method on the StackingRegressor will fit clones of those original regressors that will be stored in the class attribute self.regr_ . meta_regressor : object The meta-regressor to be fitted on the ensemble of regressors verbose : int, optional (default=0) Controls the verbosity of the building process. - verbose=0 (default): Prints nothing - verbose=1 : Prints the number & name of the regressor being fitted - verbose=2 : Prints info about the parameters of the regressor being fitted - verbose>2 : Changes verbose param of the underlying regressor to self.verbose - 2 use_features_in_secondary : bool (default: False) If True, the meta-regressor will be trained both on the predictions of the original regressors and the original dataset. If False, the meta-regressor will be trained only on the predictions of the original regressors. store_train_meta_features : bool (default: False) If True, the meta-features computed from the training data used for fitting the meta-regressor stored in the self.train_meta_features_ array, which can be accessed after calling fit . Attributes regr_ : list, shape=[n_regressors] Fitted regressors (clones of the original regressors) meta_regr_ : estimator Fitted meta-regressor (clone of the original meta-estimator) coef_ : array-like, shape = [n_features] Model coefficients of the fitted meta-estimator intercept_ : float Intercept of the fitted meta-estimator train_meta_features : numpy array, shape = [n_samples, len(self.regressors)] meta-features for training data, where n_samples is the number of samples in training data and len(self.regressors) is the number of regressors. refit : bool (default: True) Clones the regressors for stacking regression if True (default) or else uses the original ones, which will be refitted on the dataset upon calling the fit method. Setting refit=False is recommended if you are working with estimators that are supporting the scikit-learn fit/predict API interface but are not compatible to scikit-learn's clone function. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/regressor/StackingRegressor/ Methods fit(X, y, sample_weight=None) Learn weight coefficients from training data for each regressor. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : numpy array, shape = [n_samples] or [n_samples, n_targets] Target values. Multiple targets are supported only if self.multi_output is True. sample_weight : array-like, shape = [n_samples], optional Sample weights passed as sample_weights to each regressor in the regressors list as well as the meta_regressor. Raises error if some regressor does not support sample_weight in the fit() method. Returns self : object fit_transform(X, y=None, fit_params) Fit to data, then transform it. Fits transformer to `X` and `y` with optional parameters `fit_params` and returns a transformed version of `X`. Parameters X : array-like of shape (n_samples, n_features) Input samples. y : array-like of shape (n_samples,) or (n_samples, n_outputs), default=None Target values (None for unsupervised transformations). **fit_params : dict Additional fit parameters. Returns X_new : ndarray array of shape (n_samples, n_features_new) Transformed array. get_params(deep=True) Return estimator parameter names for GridSearch support. predict(X) Predict target values for X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns y_target : array-like, shape = [n_samples] or [n_samples, n_targets] Predicted target values. predict_meta_features(X) Get meta-features of test-data. Parameters X : numpy array, shape = [n_samples, n_features] Test vectors, where n_samples is the number of samples and n_features is the number of features. Returns meta-features : numpy array, shape = [n_samples, len(self.regressors)] meta-features for test data, where n_samples is the number of samples in test data and len(self.regressors) is the number of regressors. If self.multi_output is True, then the number of columns is len(self.regressors) * n_targets score(X, y, sample_weight=None) Return the coefficient of determination :math: R^2 of the prediction. The coefficient :math:`R^2` is defined as :math:`(1 - \\frac{u}{v})`, where :math:`u` is the residual sum of squares ``((y_true - y_pred) ** 2).sum() and :math:`v` is the total sum of squares ((y_true - y_true.mean()) ** 2).sum()``. The best possible score is 1.0 and it can be negative (because the model can be arbitrarily worse). A constant model that always predicts the expected value of y , disregarding the input features, would get a :math: R^2 score of 0.0. Parameters X : array-like of shape (n_samples, n_features) Test samples. For some estimators this may be a precomputed kernel matrix or a list of generic objects instead with shape (n_samples, n_samples_fitted) , where n_samples_fitted is the number of samples used in the fitting for the estimator. y : array-like of shape (n_samples,) or (n_samples, n_outputs) True values for X . sample_weight : array-like of shape (n_samples,), default=None Sample weights. Returns score : float :math: R^2 of self.predict(X) wrt. y . Notes The :math: R^2 score used when calling score on a regressor uses multioutput='uniform_average' from version 0.23 to keep consistent with default value of :func: ~sklearn.metrics.r2_score . This influences the score method of all the multioutput regressors (except for :class: ~sklearn.multioutput.MultiOutputRegressor ). set_params( params) Set the parameters of this estimator. Valid parameter keys can be listed with ``get_params()``. Returns self Properties coef_ None intercept_ None named_regressors None ython","title":"StackingRegressor: a simple stacking implementation for regression"},{"location":"user_guide/regressor/StackingRegressor/#stackingregressor-a-simple-stacking-implementation-for-regression","text":"An ensemble-learning meta-regressor for stacking regression from mlxtend.regressor import StackingRegressor","title":"StackingRegressor: a simple stacking implementation for regression"},{"location":"user_guide/regressor/StackingRegressor/#overview","text":"Stacking regression is an ensemble learning technique to combine multiple regression models via a meta-regressor. The individual regression models are trained based on the complete training set; then, the meta-regressor is fitted based on the outputs -- meta-features -- of the individual regression models in the ensemble.","title":"Overview"},{"location":"user_guide/regressor/StackingRegressor/#references","text":"Breiman, Leo. \" Stacked regressions. \" Machine learning 24.1 (1996): 49-64.","title":"References"},{"location":"user_guide/regressor/StackingRegressor/#example-1-simple-stacked-regression","text":"from mlxtend.regressor import StackingRegressor from mlxtend.data import boston_housing_data from sklearn.linear_model import LinearRegression from sklearn.linear_model import Ridge from sklearn.svm import SVR import matplotlib.pyplot as plt import numpy as np import warnings warnings.simplefilter('ignore') # Generating a sample dataset np.random.seed(1) X = np.sort(5 * np.random.rand(40, 1), axis=0) y = np.sin(X).ravel() y[::5] += 3 * (0.5 - np.random.rand(8)) # Initializing models lr = LinearRegression() svr_lin = SVR(kernel='linear') ridge = Ridge(random_state=1) svr_rbf = SVR(kernel='rbf') stregr = StackingRegressor(regressors=[svr_lin, lr, ridge], meta_regressor=svr_rbf) # Training the stacking classifier stregr.fit(X, y) stregr.predict(X) # Evaluate and visualize the fit print(\"Mean Squared Error: %.4f\" % np.mean((stregr.predict(X) - y) ** 2)) print('Variance Score: %.4f' % stregr.score(X, y)) with plt.style.context(('seaborn-whitegrid')): plt.scatter(X, y, c='lightgray') plt.plot(X, stregr.predict(X), c='darkgreen', lw=2) plt.show() Mean Squared Error: 0.1846 Variance Score: 0.7329 stregr StackingRegressor(meta_regressor=SVR(), regressors=[SVR(kernel='linear'), LinearRegression(), Ridge(random_state=1)])","title":"Example 1 - Simple Stacked Regression"},{"location":"user_guide/regressor/StackingRegressor/#example-2-stacked-regression-and-gridsearch","text":"In this second example we demonstrate how StackingCVRegressor works in combination with GridSearchCV . The stack still allows tuning hyper parameters of the base and meta models! For instance, we can use estimator.get_params().keys() to get a full list of tunable parameters. from sklearn.model_selection import GridSearchCV from sklearn.linear_model import Lasso # Initializing models lr = LinearRegression() svr_lin = SVR(kernel='linear') ridge = Ridge(random_state=1) lasso = Lasso(random_state=1) svr_rbf = SVR(kernel='rbf') regressors = [svr_lin, lr, ridge, lasso] stregr = StackingRegressor(regressors=regressors, meta_regressor=svr_rbf) params = {'lasso__alpha': [0.1, 1.0, 10.0], 'ridge__alpha': [0.1, 1.0, 10.0], 'svr__C': [0.1, 1.0, 10.0], 'meta_regressor__C': [0.1, 1.0, 10.0, 100.0], 'meta_regressor__gamma': [0.1, 1.0, 10.0]} grid = GridSearchCV(estimator=stregr, param_grid=params, cv=5, refit=True) grid.fit(X, y) print(\"Best: %f using %s\" % (grid.best_score_, grid.best_params_)) Best: -0.082717 using {'lasso__alpha': 0.1, 'meta_regressor__C': 1.0, 'meta_regressor__gamma': 1.0, 'ridge__alpha': 0.1, 'svr__C': 10.0} cv_keys = ('mean_test_score', 'std_test_score', 'params') for r, _ in enumerate(grid.cv_results_['mean_test_score']): print(\"%0.3f +/- %0.2f %r\" % (grid.cv_results_[cv_keys[0]][r], grid.cv_results_[cv_keys[1]][r] / 2.0, grid.cv_results_[cv_keys[2]][r])) if r > 10: break print('...') print('Best parameters: %s' % grid.best_params_) print('Accuracy: %.2f' % grid.best_score_) -9.810 +/- 6.86 {'lasso__alpha': 0.1, 'meta_regressor__C': 0.1, 'meta_regressor__gamma': 0.1, 'ridge__alpha': 0.1, 'svr__C': 0.1} -9.591 +/- 6.67 {'lasso__alpha': 0.1, 'meta_regressor__C': 0.1, 'meta_regressor__gamma': 0.1, 'ridge__alpha': 0.1, 'svr__C': 1.0} -9.591 +/- 6.67 {'lasso__alpha': 0.1, 'meta_regressor__C': 0.1, 'meta_regressor__gamma': 0.1, 'ridge__alpha': 0.1, 'svr__C': 10.0} -9.819 +/- 6.87 {'lasso__alpha': 0.1, 'meta_regressor__C': 0.1, 'meta_regressor__gamma': 0.1, 'ridge__alpha': 1.0, 'svr__C': 0.1} -9.600 +/- 6.68 {'lasso__alpha': 0.1, 'meta_regressor__C': 0.1, 'meta_regressor__gamma': 0.1, 'ridge__alpha': 1.0, 'svr__C': 1.0} -9.600 +/- 6.68 {'lasso__alpha': 0.1, 'meta_regressor__C': 0.1, 'meta_regressor__gamma': 0.1, 'ridge__alpha': 1.0, 'svr__C': 10.0} -9.878 +/- 6.91 {'lasso__alpha': 0.1, 'meta_regressor__C': 0.1, 'meta_regressor__gamma': 0.1, 'ridge__alpha': 10.0, 'svr__C': 0.1} -9.665 +/- 6.71 {'lasso__alpha': 0.1, 'meta_regressor__C': 0.1, 'meta_regressor__gamma': 0.1, 'ridge__alpha': 10.0, 'svr__C': 1.0} -9.665 +/- 6.71 {'lasso__alpha': 0.1, 'meta_regressor__C': 0.1, 'meta_regressor__gamma': 0.1, 'ridge__alpha': 10.0, 'svr__C': 10.0} -4.839 +/- 3.98 {'lasso__alpha': 0.1, 'meta_regressor__C': 0.1, 'meta_regressor__gamma': 1.0, 'ridge__alpha': 0.1, 'svr__C': 0.1} -3.986 +/- 3.16 {'lasso__alpha': 0.1, 'meta_regressor__C': 0.1, 'meta_regressor__gamma': 1.0, 'ridge__alpha': 0.1, 'svr__C': 1.0} -3.986 +/- 3.16 {'lasso__alpha': 0.1, 'meta_regressor__C': 0.1, 'meta_regressor__gamma': 1.0, 'ridge__alpha': 0.1, 'svr__C': 10.0} ... Best parameters: {'lasso__alpha': 0.1, 'meta_regressor__C': 1.0, 'meta_regressor__gamma': 1.0, 'ridge__alpha': 0.1, 'svr__C': 10.0} Accuracy: -0.08 # Evaluate and visualize the fit print(\"Mean Squared Error: %.4f\" % np.mean((grid.predict(X) - y) ** 2)) print('Variance Score: %.4f' % grid.score(X, y)) with plt.style.context(('seaborn-whitegrid')): plt.scatter(X, y, c='lightgray') plt.plot(X, grid.predict(X), c='darkgreen', lw=2) plt.show() Mean Squared Error: 0.1845 Variance Score: 0.7330 Note The StackingCVRegressor also enables grid search over the regressors and even a single base regressor. When there are level-mixed hyperparameters, GridSearchCV will try to replace hyperparameters in a top-down order, i.e., regressors -> single base regressor -> regressor hyperparameter. For instance, given a hyperparameter grid such as params = {'randomforestregressor__n_estimators': [1, 100], 'regressors': [(regr1, regr1, regr1), (regr2, regr3)]} it will first use the instance settings of either (regr1, regr2, regr3) or (regr2, regr3) . Then it will replace the 'n_estimators' settings for a matching regressor based on 'randomforestregressor__n_estimators': [1, 100] .","title":"Example 2 - Stacked Regression and GridSearch"},{"location":"user_guide/regressor/StackingRegressor/#api","text":"StackingRegressor(regressors, meta_regressor, verbose=0, use_features_in_secondary=False, store_train_meta_features=False, refit=True, multi_output=False) A Stacking regressor for scikit-learn estimators for regression. Parameters regressors : array-like, shape = [n_regressors] A list of regressors. Invoking the fit method on the StackingRegressor will fit clones of those original regressors that will be stored in the class attribute self.regr_ . meta_regressor : object The meta-regressor to be fitted on the ensemble of regressors verbose : int, optional (default=0) Controls the verbosity of the building process. - verbose=0 (default): Prints nothing - verbose=1 : Prints the number & name of the regressor being fitted - verbose=2 : Prints info about the parameters of the regressor being fitted - verbose>2 : Changes verbose param of the underlying regressor to self.verbose - 2 use_features_in_secondary : bool (default: False) If True, the meta-regressor will be trained both on the predictions of the original regressors and the original dataset. If False, the meta-regressor will be trained only on the predictions of the original regressors. store_train_meta_features : bool (default: False) If True, the meta-features computed from the training data used for fitting the meta-regressor stored in the self.train_meta_features_ array, which can be accessed after calling fit . Attributes regr_ : list, shape=[n_regressors] Fitted regressors (clones of the original regressors) meta_regr_ : estimator Fitted meta-regressor (clone of the original meta-estimator) coef_ : array-like, shape = [n_features] Model coefficients of the fitted meta-estimator intercept_ : float Intercept of the fitted meta-estimator train_meta_features : numpy array, shape = [n_samples, len(self.regressors)] meta-features for training data, where n_samples is the number of samples in training data and len(self.regressors) is the number of regressors. refit : bool (default: True) Clones the regressors for stacking regression if True (default) or else uses the original ones, which will be refitted on the dataset upon calling the fit method. Setting refit=False is recommended if you are working with estimators that are supporting the scikit-learn fit/predict API interface but are not compatible to scikit-learn's clone function. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/regressor/StackingRegressor/","title":"API"},{"location":"user_guide/regressor/StackingRegressor/#methods","text":"fit(X, y, sample_weight=None) Learn weight coefficients from training data for each regressor. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. y : numpy array, shape = [n_samples] or [n_samples, n_targets] Target values. Multiple targets are supported only if self.multi_output is True. sample_weight : array-like, shape = [n_samples], optional Sample weights passed as sample_weights to each regressor in the regressors list as well as the meta_regressor. Raises error if some regressor does not support sample_weight in the fit() method. Returns self : object fit_transform(X, y=None, fit_params) Fit to data, then transform it. Fits transformer to `X` and `y` with optional parameters `fit_params` and returns a transformed version of `X`. Parameters X : array-like of shape (n_samples, n_features) Input samples. y : array-like of shape (n_samples,) or (n_samples, n_outputs), default=None Target values (None for unsupervised transformations). **fit_params : dict Additional fit parameters. Returns X_new : ndarray array of shape (n_samples, n_features_new) Transformed array. get_params(deep=True) Return estimator parameter names for GridSearch support. predict(X) Predict target values for X. Parameters X : {array-like, sparse matrix}, shape = [n_samples, n_features] Training vectors, where n_samples is the number of samples and n_features is the number of features. Returns y_target : array-like, shape = [n_samples] or [n_samples, n_targets] Predicted target values. predict_meta_features(X) Get meta-features of test-data. Parameters X : numpy array, shape = [n_samples, n_features] Test vectors, where n_samples is the number of samples and n_features is the number of features. Returns meta-features : numpy array, shape = [n_samples, len(self.regressors)] meta-features for test data, where n_samples is the number of samples in test data and len(self.regressors) is the number of regressors. If self.multi_output is True, then the number of columns is len(self.regressors) * n_targets score(X, y, sample_weight=None) Return the coefficient of determination :math: R^2 of the prediction. The coefficient :math:`R^2` is defined as :math:`(1 - \\frac{u}{v})`, where :math:`u` is the residual sum of squares ``((y_true - y_pred) ** 2).sum() and :math:`v` is the total sum of squares ((y_true - y_true.mean()) ** 2).sum()``. The best possible score is 1.0 and it can be negative (because the model can be arbitrarily worse). A constant model that always predicts the expected value of y , disregarding the input features, would get a :math: R^2 score of 0.0. Parameters X : array-like of shape (n_samples, n_features) Test samples. For some estimators this may be a precomputed kernel matrix or a list of generic objects instead with shape (n_samples, n_samples_fitted) , where n_samples_fitted is the number of samples used in the fitting for the estimator. y : array-like of shape (n_samples,) or (n_samples, n_outputs) True values for X . sample_weight : array-like of shape (n_samples,), default=None Sample weights. Returns score : float :math: R^2 of self.predict(X) wrt. y . Notes The :math: R^2 score used when calling score on a regressor uses multioutput='uniform_average' from version 0.23 to keep consistent with default value of :func: ~sklearn.metrics.r2_score . This influences the score method of all the multioutput regressors (except for :class: ~sklearn.multioutput.MultiOutputRegressor ). set_params( params) Set the parameters of this estimator. Valid parameter keys can be listed with ``get_params()``. Returns self","title":"Methods"},{"location":"user_guide/regressor/StackingRegressor/#properties","text":"coef_ None intercept_ None named_regressors None ython","title":"Properties"},{"location":"user_guide/text/generalize_names/","text":"generalize_names: convert names into a generalized format A function that converts a name into a general format <last_name><separator><firstname letter(s)> (all lowercase) . from mlxtend.text import generalize_names Overview A function that converts a name into a general format <last_name><separator><firstname letter(s)> (all lowercase) , which is useful if data is collected from different sources and is supposed to be compared or merged based on name identifiers. E.g., if names are stored in a pandas DataFrame column, the apply function can be used to generalize names: df['name'] = df['name'].apply(generalize_names) References - Example 1 - Defaults from mlxtend.text import generalize_names generalize_names('Pozo, Jos\u00e9 \u00c1ngel') 'pozo j' generalize_names('Jos\u00e9 Pozo') 'pozo j' generalize_names('Jos\u00e9 \u00c1ngel Pozo') 'pozo j' Example 2 - Optional Parameters from mlxtend.text import generalize_names generalize_names(\"Eto'o, Samuel\", firstname_output_letters=2) 'etoo sa' generalize_names(\"Eto'o, Samuel\", firstname_output_letters=0) 'etoo' generalize_names(\"Eto'o, Samuel\", output_sep=', ') 'etoo, s' API generalize_names(name, output_sep=' ', firstname_output_letters=1) Generalize a person's first and last name. Returns a person's name in the format <last_name><separator><firstname letter(s)> (all lowercase) Parameters name : str Name of the player output_sep : str (default: ' ') String for separating last name and first name in the output. firstname_output_letters : int Number of letters in the abbreviated first name. Returns gen_name : str The generalized name. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/text/generalize_names/","title":"generalize_names: convert names into a generalized format"},{"location":"user_guide/text/generalize_names/#generalize_names-convert-names-into-a-generalized-format","text":"A function that converts a name into a general format <last_name><separator><firstname letter(s)> (all lowercase) . from mlxtend.text import generalize_names","title":"generalize_names: convert names into a generalized format"},{"location":"user_guide/text/generalize_names/#overview","text":"A function that converts a name into a general format <last_name><separator><firstname letter(s)> (all lowercase) , which is useful if data is collected from different sources and is supposed to be compared or merged based on name identifiers. E.g., if names are stored in a pandas DataFrame column, the apply function can be used to generalize names: df['name'] = df['name'].apply(generalize_names)","title":"Overview"},{"location":"user_guide/text/generalize_names/#references","text":"-","title":"References"},{"location":"user_guide/text/generalize_names/#example-1-defaults","text":"from mlxtend.text import generalize_names generalize_names('Pozo, Jos\u00e9 \u00c1ngel') 'pozo j' generalize_names('Jos\u00e9 Pozo') 'pozo j' generalize_names('Jos\u00e9 \u00c1ngel Pozo') 'pozo j'","title":"Example 1 - Defaults"},{"location":"user_guide/text/generalize_names/#example-2-optional-parameters","text":"from mlxtend.text import generalize_names generalize_names(\"Eto'o, Samuel\", firstname_output_letters=2) 'etoo sa' generalize_names(\"Eto'o, Samuel\", firstname_output_letters=0) 'etoo' generalize_names(\"Eto'o, Samuel\", output_sep=', ') 'etoo, s'","title":"Example 2 - Optional Parameters"},{"location":"user_guide/text/generalize_names/#api","text":"generalize_names(name, output_sep=' ', firstname_output_letters=1) Generalize a person's first and last name. Returns a person's name in the format <last_name><separator><firstname letter(s)> (all lowercase) Parameters name : str Name of the player output_sep : str (default: ' ') String for separating last name and first name in the output. firstname_output_letters : int Number of letters in the abbreviated first name. Returns gen_name : str The generalized name. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/text/generalize_names/","title":"API"},{"location":"user_guide/text/generalize_names_duplcheck/","text":"generalize_names_duplcheck: Generalize names while preventing duplicates among different names A function that converts a name into a general format <last_name><separator><firstname letter(s)> (all lowercase) in a pandas DataFrame while avoiding duplicate entries. from mlxtend.text import generalize_names_duplcheck Overview Note that using mlxtend.text.generalize_names with few firstname_output_letters can result in duplicate entries. E.g., if your dataset contains the names \"Adam Johnson\" and \"Andrew Johnson\", the default setting (i.e., 1 first name letter) will produce the generalized name \"johnson a\" in both cases. One solution is to increase the number of first name letters in the output by setting the parameter firstname_output_letters to a value larger than 1. An alternative solution is to use the generalize_names_duplcheck function if you are working with pandas DataFrames. By default, generalize_names_duplcheck will apply generalize_names to a pandas DataFrame column with the minimum number of first name letters and append as many first name letters as necessary until no duplicates are present in the given DataFrame column. An example dataset column that contains the names References - Example 1 - Defaults Reading in a CSV file that has column Name for which we want to generalize the names: Samuel Eto'o Adam Johnson Andrew Johnson import pandas as pd from io import StringIO simulated_csv = \"name,some_value\\n\"\\ \"Samuel Eto'o,1\\n\"\\ \"Adam Johnson,1\\n\"\\ \"Andrew Johnson,1\\n\" df = pd.read_csv(StringIO(simulated_csv)) df name some_value 0 Samuel Eto'o 1 1 Adam Johnson 1 2 Andrew Johnson 1 Applying generalize_names_duplcheck to generate a new DataFrame with the generalized names without duplicates: from mlxtend.text import generalize_names_duplcheck df_new = generalize_names_duplcheck(df=df, col_name='name') df_new name some_value 0 etoo s 1 1 johnson ad 1 2 johnson an 1 API generalize_names_duplcheck(df, col_name) Generalizes names and removes duplicates. Applies mlxtend.text.generalize_names to a DataFrame with 1 first name letter by default and uses more first name letters if duplicates are detected. Parameters df : pandas.DataFrame DataFrame that contains a column where generalize_names should be applied. col_name : str Name of the DataFrame column where generalize_names function should be applied to. Returns df_new : str New DataFrame object where generalize_names function has been applied without duplicates. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/text/generalize_names_duplcheck/","title":"generalize_names_duplcheck: Generalize names while preventing duplicates among different names"},{"location":"user_guide/text/generalize_names_duplcheck/#generalize_names_duplcheck-generalize-names-while-preventing-duplicates-among-different-names","text":"A function that converts a name into a general format <last_name><separator><firstname letter(s)> (all lowercase) in a pandas DataFrame while avoiding duplicate entries. from mlxtend.text import generalize_names_duplcheck","title":"generalize_names_duplcheck: Generalize names while preventing duplicates among different names"},{"location":"user_guide/text/generalize_names_duplcheck/#overview","text":"Note that using mlxtend.text.generalize_names with few firstname_output_letters can result in duplicate entries. E.g., if your dataset contains the names \"Adam Johnson\" and \"Andrew Johnson\", the default setting (i.e., 1 first name letter) will produce the generalized name \"johnson a\" in both cases. One solution is to increase the number of first name letters in the output by setting the parameter firstname_output_letters to a value larger than 1. An alternative solution is to use the generalize_names_duplcheck function if you are working with pandas DataFrames. By default, generalize_names_duplcheck will apply generalize_names to a pandas DataFrame column with the minimum number of first name letters and append as many first name letters as necessary until no duplicates are present in the given DataFrame column. An example dataset column that contains the names","title":"Overview"},{"location":"user_guide/text/generalize_names_duplcheck/#references","text":"-","title":"References"},{"location":"user_guide/text/generalize_names_duplcheck/#example-1-defaults","text":"Reading in a CSV file that has column Name for which we want to generalize the names: Samuel Eto'o Adam Johnson Andrew Johnson import pandas as pd from io import StringIO simulated_csv = \"name,some_value\\n\"\\ \"Samuel Eto'o,1\\n\"\\ \"Adam Johnson,1\\n\"\\ \"Andrew Johnson,1\\n\" df = pd.read_csv(StringIO(simulated_csv)) df name some_value 0 Samuel Eto'o 1 1 Adam Johnson 1 2 Andrew Johnson 1 Applying generalize_names_duplcheck to generate a new DataFrame with the generalized names without duplicates: from mlxtend.text import generalize_names_duplcheck df_new = generalize_names_duplcheck(df=df, col_name='name') df_new name some_value 0 etoo s 1 1 johnson ad 1 2 johnson an 1","title":"Example 1 - Defaults"},{"location":"user_guide/text/generalize_names_duplcheck/#api","text":"generalize_names_duplcheck(df, col_name) Generalizes names and removes duplicates. Applies mlxtend.text.generalize_names to a DataFrame with 1 first name letter by default and uses more first name letters if duplicates are detected. Parameters df : pandas.DataFrame DataFrame that contains a column where generalize_names should be applied. col_name : str Name of the DataFrame column where generalize_names function should be applied to. Returns df_new : str New DataFrame object where generalize_names function has been applied without duplicates. Examples For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/text/generalize_names_duplcheck/","title":"API"},{"location":"user_guide/text/tokenizer/","text":"tokenizer_emoticons: tokenizers for emoticons Different functions to tokenize text. from mlxtend.text import tokenizer_emoticons from mlxtend.text import tokenizer_words_and_emoticons Overview Different functions to tokenize text for natural language processing tasks, for example such as building a bag-of-words model for text classification. References - Example 1 - Extract Emoticons from mlxtend.text import tokenizer_emoticons tokenizer_emoticons('</a>This :) is :( a test :-)!') [':)', ':(', ':-)'] Example 2 - Extract Words and Emoticons from mlxtend.text import tokenizer_words_and_emoticons tokenizer_words_and_emoticons('</a>This :) is :( a test :-)!') ['this', 'is', 'a', 'test', ':)', ':(', ':-)'] API tokenizer_emoticons(text) Return emoticons from text Examples >>> tokenizer_emoticons('</a>This :) is :( a test :-)!') [':)', ':(', ':-)'] For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/text/tokenizer_emoticons/ tokenizer_words_and_emoticons(text) Convert text to lowercase words and emoticons. Examples >>> tokenizer_words_and_emoticons('</a>This :) is :( a test :-)!') ['this', 'is', 'a', 'test', ':)', ':(', ':-)'] For more usage examples, please see https://rasbt.github.io/mlxtend/user_guide/text/tokenizer_words_and_emoticons/","title":"tokenizer_emoticons: tokenizers for emoticons"},{"location":"user_guide/text/tokenizer/#tokenizer_emoticons-tokenizers-for-emoticons","text":"Different functions to tokenize text. from mlxtend.text import tokenizer_emoticons from mlxtend.text import tokenizer_words_and_emoticons","title":"tokenizer_emoticons: tokenizers for emoticons"},{"location":"user_guide/text/tokenizer/#overview","text":"Different functions to tokenize text for natural language processing tasks, for example such as building a bag-of-words model for text classification.","title":"Overview"},{"location":"user_guide/text/tokenizer/#references","text":"-","title":"References"},{"location":"user_guide/text/tokenizer/#example-1-extract-emoticons","text":"from mlxtend.text import tokenizer_emoticons tokenizer_emoticons('</a>This :) is :( a test :-)!') [':)', ':(', ':-)']","title":"Example 1 - Extract Emoticons"},{"location":"user_guide/text/tokenizer/#example-2-extract-words-and-emoticons","text":"from mlxtend.text import tokenizer_words_and_emoticons tokenizer_words_and_emoticons('</a>This :) is :( a test :-)!') ['this', 'is', 'a', 'test', ':)', ':(', ':-)']","title":"Example 2 - Extract Words and Emoticons"},{"location":"user_guide/text/tokenizer/#api","text":"tokenizer_emoticons(text) Return emoticons from text Examples >>> tokenizer_emoticons('</a>This :) is :( a test :-)!') [':)', ':(', ':-)'] For usage examples, please see https://rasbt.github.io/mlxtend/user_guide/text/tokenizer_emoticons/ tokenizer_words_and_emoticons(text) Convert text to lowercase words and emoticons. Examples >>> tokenizer_words_and_emoticons('</a>This :) is :( a test :-)!') ['this', 'is', 'a', 'test', ':)', ':(', ':-)'] For more usage examples, please see https://rasbt.github.io/mlxtend/user_guide/text/tokenizer_words_and_emoticons/","title":"API"},{"location":"user_guide/utils/Counter/","text":"Counter: A simple progress counter A simple progress counter to print the number of iterations and time elapsed in a for-loop execution. from mlxtend.utils import Counter Overview The Counter class implements an object for displaying the number of iterations and time elapsed in a for-loop. Please note that the Counter was implemented for efficiency; thus, the Counter offers only very basic functionality in order to avoid relatively expensive evaluations (of if-else statements). References - Example 1 - Counting the iterations in a for-loop from mlxtend.utils import Counter import time cnt = Counter() for i in range(20): # do some computation time.sleep(0.1) cnt.update() 20 iter | 2 sec Note that the first number displays the current iteration, and the second number shows the time elapsed after initializing the Counter . API Counter(stderr=False, start_newline=True, precision=0, name=None) Class to display the progress of for-loop iterators. Parameters stderr : bool (default: True) Prints output to sys.stderr if True; uses sys.stdout otherwise. start_newline : bool (default: True) Prepends a new line to the counter, which prevents overwriting counters if multiple counters are printed in succession. precision: int (default: 0) Sets the number of decimal places when displaying the time elapsed in seconds. name : string (default: None) Prepends the specified name before the counter to allow distinguishing between multiple counters. Attributes curr_iter : int The current iteration. start_time : float The system's time in seconds when the Counter was initialized. end_time : float The system's time in seconds when the Counter was last updated. Examples >>> cnt = Counter() >>> for i in range(20): ... # do some computation ... time.sleep(0.1) ... cnt.update() 20 iter | 2 sec >>> print('The counter was initialized.' ' %d seconds ago.' % (time.time() - cnt.start_time)) The counter was initialized 2 seconds ago >>> print('The counter was last updated' ' %d seconds ago.' % (time.time() - cnt.end_time)) The counter was last updated 0 seconds ago. For more usage examples, please see https://rasbt.github.io/mlxtend/user_guide/utils/Counter/ Methods update() Print current iteration and time elapsed.","title":"Counter: A simple progress counter"},{"location":"user_guide/utils/Counter/#counter-a-simple-progress-counter","text":"A simple progress counter to print the number of iterations and time elapsed in a for-loop execution. from mlxtend.utils import Counter","title":"Counter: A simple progress counter"},{"location":"user_guide/utils/Counter/#overview","text":"The Counter class implements an object for displaying the number of iterations and time elapsed in a for-loop. Please note that the Counter was implemented for efficiency; thus, the Counter offers only very basic functionality in order to avoid relatively expensive evaluations (of if-else statements).","title":"Overview"},{"location":"user_guide/utils/Counter/#references","text":"-","title":"References"},{"location":"user_guide/utils/Counter/#example-1-counting-the-iterations-in-a-for-loop","text":"from mlxtend.utils import Counter import time cnt = Counter() for i in range(20): # do some computation time.sleep(0.1) cnt.update() 20 iter | 2 sec Note that the first number displays the current iteration, and the second number shows the time elapsed after initializing the Counter .","title":"Example 1 - Counting the iterations in a for-loop"},{"location":"user_guide/utils/Counter/#api","text":"Counter(stderr=False, start_newline=True, precision=0, name=None) Class to display the progress of for-loop iterators. Parameters stderr : bool (default: True) Prints output to sys.stderr if True; uses sys.stdout otherwise. start_newline : bool (default: True) Prepends a new line to the counter, which prevents overwriting counters if multiple counters are printed in succession. precision: int (default: 0) Sets the number of decimal places when displaying the time elapsed in seconds. name : string (default: None) Prepends the specified name before the counter to allow distinguishing between multiple counters. Attributes curr_iter : int The current iteration. start_time : float The system's time in seconds when the Counter was initialized. end_time : float The system's time in seconds when the Counter was last updated. Examples >>> cnt = Counter() >>> for i in range(20): ... # do some computation ... time.sleep(0.1) ... cnt.update() 20 iter | 2 sec >>> print('The counter was initialized.' ' %d seconds ago.' % (time.time() - cnt.start_time)) The counter was initialized 2 seconds ago >>> print('The counter was last updated' ' %d seconds ago.' % (time.time() - cnt.end_time)) The counter was last updated 0 seconds ago. For more usage examples, please see https://rasbt.github.io/mlxtend/user_guide/utils/Counter/","title":"API"},{"location":"user_guide/utils/Counter/#methods","text":"update() Print current iteration and time elapsed.","title":"Methods"}]}
\ No newline at end of file
diff --git a/sitemap.xml b/sitemap.xml
index ef2e64746..284dbaf4f 100644
--- a/sitemap.xml
+++ b/sitemap.xml
@@ -2,1072 +2,1072 @@
 <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
     <url>
          <loc>https://rasbt.github.io/mlxtend/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/CHANGELOG/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/CONTRIBUTING/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/Code-of-Conduct/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/USER-GUIDE-INDEX/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/USER_GUIDE_INDEX/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/cite/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/contributors/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/discuss/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/installation/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/license/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/api_modules/mlxtend.classifier/Adaline/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/api_modules/mlxtend.classifier/EnsembleVoteClassifier/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/api_modules/mlxtend.classifier/LogisticRegression/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/api_modules/mlxtend.classifier/MultiLayerPerceptron/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/api_modules/mlxtend.classifier/OneRClassifier/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/api_modules/mlxtend.classifier/Perceptron/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/api_modules/mlxtend.classifier/SoftmaxRegression/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/api_modules/mlxtend.classifier/StackingCVClassifier/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/api_modules/mlxtend.classifier/StackingClassifier/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/api_modules/mlxtend.cluster/Kmeans/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/api_modules/mlxtend.data/autompg_data/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/api_modules/mlxtend.data/boston_housing_data/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/api_modules/mlxtend.data/iris_data/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/api_modules/mlxtend.data/loadlocal_mnist/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/api_modules/mlxtend.data/make_multiplexer_dataset/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/api_modules/mlxtend.data/mnist_data/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/api_modules/mlxtend.data/three_blobs_data/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/api_modules/mlxtend.data/wine_data/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/api_modules/mlxtend.evaluate/BootstrapOutOfBag/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/api_modules/mlxtend.evaluate/GroupTimeSeriesSplit/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/api_modules/mlxtend.evaluate/PredefinedHoldoutSplit/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/api_modules/mlxtend.evaluate/RandomHoldoutSplit/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/api_modules/mlxtend.evaluate/accuracy_score/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/api_modules/mlxtend.evaluate/bias_variance_decomp/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/api_modules/mlxtend.evaluate/bootstrap/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/api_modules/mlxtend.evaluate/bootstrap_point632_score/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/api_modules/mlxtend.evaluate/cochrans_q/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/api_modules/mlxtend.evaluate/combined_ftest_5x2cv/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/api_modules/mlxtend.evaluate/confusion_matrix/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/api_modules/mlxtend.evaluate/create_counterfactual/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/api_modules/mlxtend.evaluate/feature_importance_permutation/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/api_modules/mlxtend.evaluate/ftest/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/api_modules/mlxtend.evaluate/lift_score/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/api_modules/mlxtend.evaluate/mcnemar/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/api_modules/mlxtend.evaluate/mcnemar_table/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/api_modules/mlxtend.evaluate/mcnemar_tables/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/api_modules/mlxtend.evaluate/paired_ttest_5x2cv/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/api_modules/mlxtend.evaluate/paired_ttest_kfold_cv/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/api_modules/mlxtend.evaluate/paired_ttest_resampled/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/api_modules/mlxtend.evaluate/permutation_test/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/api_modules/mlxtend.evaluate/proportion_difference/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/api_modules/mlxtend.evaluate/scoring/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/api_modules/mlxtend.feature_extraction/LinearDiscriminantAnalysis/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/api_modules/mlxtend.feature_extraction/PrincipalComponentAnalysis/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/api_modules/mlxtend.feature_extraction/RBFKernelPCA/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/api_modules/mlxtend.feature_selection/ColumnSelector/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/api_modules/mlxtend.feature_selection/ExhaustiveFeatureSelector/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/api_modules/mlxtend.feature_selection/SequentialFeatureSelector/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/api_modules/mlxtend.file_io/find_filegroups/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/api_modules/mlxtend.file_io/find_files/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/api_modules/mlxtend.frequent_patterns/apriori/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/api_modules/mlxtend.frequent_patterns/association_rules/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/api_modules/mlxtend.frequent_patterns/fpgrowth/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/api_modules/mlxtend.frequent_patterns/fpmax/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/api_modules/mlxtend.frequent_patterns/hmine/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/api_modules/mlxtend.image/EyepadAlign/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/api_modules/mlxtend.image/extract_face_landmarks/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/api_modules/mlxtend.math/factorial/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/api_modules/mlxtend.math/num_combinations/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/api_modules/mlxtend.math/num_permutations/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/api_modules/mlxtend.math/vectorspace_dimensionality/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/api_modules/mlxtend.math/vectorspace_orthonormalization/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/api_modules/mlxtend.plotting/category_scatter/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/api_modules/mlxtend.plotting/checkerboard_plot/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/api_modules/mlxtend.plotting/ecdf/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/api_modules/mlxtend.plotting/enrichment_plot/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/api_modules/mlxtend.plotting/heatmap/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/api_modules/mlxtend.plotting/plot_confusion_matrix/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/api_modules/mlxtend.plotting/plot_decision_regions/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/api_modules/mlxtend.plotting/plot_learning_curves/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/api_modules/mlxtend.plotting/plot_linear_regression/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/api_modules/mlxtend.plotting/plot_pca_correlation_graph/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/api_modules/mlxtend.plotting/plot_sequential_feature_selection/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/api_modules/mlxtend.plotting/remove_borders/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/api_modules/mlxtend.plotting/scatter_hist/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/api_modules/mlxtend.plotting/scatterplotmatrix/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/api_modules/mlxtend.plotting/stacked_barplot/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/api_modules/mlxtend.preprocessing/CopyTransformer/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/api_modules/mlxtend.preprocessing/DenseTransformer/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/api_modules/mlxtend.preprocessing/MeanCenterer/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/api_modules/mlxtend.preprocessing/TransactionEncoder/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/api_modules/mlxtend.preprocessing/minmax_scaling/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/api_modules/mlxtend.preprocessing/one_hot/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/api_modules/mlxtend.preprocessing/shuffle_arrays_unison/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/api_modules/mlxtend.preprocessing/standardize/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/api_modules/mlxtend.regressor/LinearRegression/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/api_modules/mlxtend.regressor/StackingCVRegressor/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/api_modules/mlxtend.regressor/StackingRegressor/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/api_modules/mlxtend.text/generalize_names/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/api_modules/mlxtend.text/generalize_names_duplcheck/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/api_modules/mlxtend.text/tokenizer_emoticons/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/api_modules/mlxtend.text/tokenizer_words_and_emoticons/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/api_modules/mlxtend.utils/Counter/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/api_modules/mlxtend.utils/assert_raises/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/api_modules/mlxtend.utils/check_Xy/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/api_modules/mlxtend.utils/format_kwarg_dictionaries/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/api_subpackages/mlxtend._base/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/api_subpackages/mlxtend.classifier/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/api_subpackages/mlxtend.cluster/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/api_subpackages/mlxtend.data/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/api_subpackages/mlxtend.evaluate/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/api_subpackages/mlxtend.externals/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/api_subpackages/mlxtend.feature_extraction/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/api_subpackages/mlxtend.feature_selection/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/api_subpackages/mlxtend.file_io/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/api_subpackages/mlxtend.frequent_patterns/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/api_subpackages/mlxtend.image/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/api_subpackages/mlxtend.math/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/api_subpackages/mlxtend.plotting/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/api_subpackages/mlxtend.preprocessing/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/api_subpackages/mlxtend.regressor/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/api_subpackages/mlxtend.text/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/api_subpackages/mlxtend.utils/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/user_guide/classifier/Adaline/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/user_guide/classifier/EnsembleVoteClassifier/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/user_guide/classifier/LogisticRegression/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/user_guide/classifier/MultiLayerPerceptron/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/user_guide/classifier/OneRClassifier/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/user_guide/classifier/Perceptron/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/user_guide/classifier/SoftmaxRegression/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/user_guide/classifier/StackingCVClassifier/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/user_guide/classifier/StackingClassifier/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/user_guide/cluster/Kmeans/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/user_guide/data/autompg_data/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/user_guide/data/boston_housing_data/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/user_guide/data/iris_data/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/user_guide/data/loadlocal_mnist/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/user_guide/data/make_multiplexer_dataset/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/user_guide/data/mnist_data/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/user_guide/data/three_blobs_data/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/user_guide/data/wine_data/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/user_guide/evaluate/BootstrapOutOfBag/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/user_guide/evaluate/GroupTimeSeriesSplit/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/user_guide/evaluate/PredefinedHoldoutSplit/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/user_guide/evaluate/RandomHoldoutSplit/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/user_guide/evaluate/accuracy_score/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/user_guide/evaluate/bias_variance_decomp/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/user_guide/evaluate/bootstrap/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/user_guide/evaluate/bootstrap_point632_score/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/user_guide/evaluate/cochrans_q/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/user_guide/evaluate/combined_ftest_5x2cv/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/user_guide/evaluate/confusion_matrix/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/user_guide/evaluate/create_counterfactual/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/user_guide/evaluate/feature_importance_permutation/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/user_guide/evaluate/ftest/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/user_guide/evaluate/lift_score/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/user_guide/evaluate/mcnemar/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/user_guide/evaluate/mcnemar_table/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/user_guide/evaluate/mcnemar_tables/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/user_guide/evaluate/paired_ttest_5x2cv/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/user_guide/evaluate/paired_ttest_kfold_cv/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/user_guide/evaluate/paired_ttest_resampled/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/user_guide/evaluate/permutation_test/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/user_guide/evaluate/proportion_difference/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/user_guide/evaluate/scoring/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/user_guide/feature_extraction/LinearDiscriminantAnalysis/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/user_guide/feature_extraction/PrincipalComponentAnalysis/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/user_guide/feature_extraction/RBFKernelPCA/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/user_guide/feature_selection/ColumnSelector/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/user_guide/feature_selection/ExhaustiveFeatureSelector/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/user_guide/feature_selection/SequentialFeatureSelector/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/user_guide/file_io/find_filegroups/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/user_guide/file_io/find_files/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/user_guide/frequent_patterns/apriori/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/user_guide/frequent_patterns/association_rules/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/user_guide/frequent_patterns/fpgrowth/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/user_guide/frequent_patterns/fpmax/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/user_guide/frequent_patterns/hmine/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/user_guide/image/extract_face_landmarks/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/user_guide/image/eyepad_align/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/user_guide/math/num_combinations/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/user_guide/math/num_permutations/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/user_guide/math/vectorspace_dimensionality/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/user_guide/math/vectorspace_orthonormalization/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/user_guide/plotting/category_scatter/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/user_guide/plotting/checkerboard_plot/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/user_guide/plotting/ecdf/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/user_guide/plotting/enrichment_plot/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/user_guide/plotting/heatmap/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/user_guide/plotting/plot_confusion_matrix/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/user_guide/plotting/plot_decision_regions/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/user_guide/plotting/plot_learning_curves/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/user_guide/plotting/plot_linear_regression/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/user_guide/plotting/plot_pca_correlation_graph/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/user_guide/plotting/plot_sequential_feature_selection/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/user_guide/plotting/scatter_hist/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/user_guide/plotting/scatterplotmatrix/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/user_guide/plotting/stacked_barplot/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/user_guide/preprocessing/CopyTransformer/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/user_guide/preprocessing/DenseTransformer/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/user_guide/preprocessing/MeanCenterer/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/user_guide/preprocessing/TransactionEncoder/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/user_guide/preprocessing/minmax_scaling/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/user_guide/preprocessing/one-hot_encoding/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/user_guide/preprocessing/shuffle_arrays_unison/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/user_guide/preprocessing/standardize/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/user_guide/regressor/LinearRegression/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/user_guide/regressor/StackingCVRegressor/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/user_guide/regressor/StackingRegressor/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/user_guide/text/generalize_names/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/user_guide/text/generalize_names_duplcheck/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/user_guide/text/tokenizer/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://rasbt.github.io/mlxtend/user_guide/utils/Counter/</loc>
-         <lastmod>2024-11-05</lastmod>
+         <lastmod>2024-11-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
 </urlset>
\ No newline at end of file
diff --git a/sitemap.xml.gz b/sitemap.xml.gz
index 9f2a386d269293621a73bd21afb760a207e605e1..5d3a3ae2fcb49fa0d265150dfdf7adfd41806f32 100644
GIT binary patch
delta 30
mcmdnUzmcC^zMF&N{xq|R?AI7KZ@hP!jpO6ukNl#G7#IMuUJJqi

delta 30
mcmdnUzmcC^zMF$XK|^aI`!&YP8}FTF<LIo=j}l$PzyJV~bqSvU

diff --git a/user_guide/classifier/Adaline/index.html b/user_guide/classifier/Adaline/index.html
index 531ded372..404909bae 100644
--- a/user_guide/classifier/Adaline/index.html
+++ b/user_guide/classifier/Adaline/index.html
@@ -12,25 +12,22 @@
         <link href="../../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../../cinder/css/base.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../../evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../../evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../../evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../../evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../../evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -1028,14 +1025,16 @@ <h3 id="methods">Methods</h3>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../../mathjaxhelper.js" defer></script>
-        <script src="../../../search/main.js" defer></script>
+        <script src="../../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../../mathjaxhelper.js"></script>
+        <script src="../../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/user_guide/classifier/EnsembleVoteClassifier/index.html b/user_guide/classifier/EnsembleVoteClassifier/index.html
index 57b2986fd..5ef467399 100644
--- a/user_guide/classifier/EnsembleVoteClassifier/index.html
+++ b/user_guide/classifier/EnsembleVoteClassifier/index.html
@@ -12,25 +12,22 @@
         <link href="../../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../../cinder/css/base.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../../evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../../evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../../evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../../evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../../evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -1543,14 +1540,16 @@ <h3 id="methods">Methods</h3>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../../mathjaxhelper.js" defer></script>
-        <script src="../../../search/main.js" defer></script>
+        <script src="../../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../../mathjaxhelper.js"></script>
+        <script src="../../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/user_guide/classifier/LogisticRegression/index.html b/user_guide/classifier/LogisticRegression/index.html
index f9acbfb7e..3a070f13d 100644
--- a/user_guide/classifier/LogisticRegression/index.html
+++ b/user_guide/classifier/LogisticRegression/index.html
@@ -12,25 +12,22 @@
         <link href="../../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../../cinder/css/base.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../../evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../../evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../../evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../../evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../../evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -1092,14 +1089,16 @@ <h3 id="methods">Methods</h3>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../../mathjaxhelper.js" defer></script>
-        <script src="../../../search/main.js" defer></script>
+        <script src="../../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../../mathjaxhelper.js"></script>
+        <script src="../../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/user_guide/classifier/MultiLayerPerceptron/index.html b/user_guide/classifier/MultiLayerPerceptron/index.html
index b0c23ad81..b4d5dfc1b 100644
--- a/user_guide/classifier/MultiLayerPerceptron/index.html
+++ b/user_guide/classifier/MultiLayerPerceptron/index.html
@@ -12,25 +12,22 @@
         <link href="../../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../../cinder/css/base.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../../evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../../evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../../evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../../evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../../evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -1015,14 +1012,16 @@ <h3 id="methods">Methods</h3>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../../mathjaxhelper.js" defer></script>
-        <script src="../../../search/main.js" defer></script>
+        <script src="../../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../../mathjaxhelper.js"></script>
+        <script src="../../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/user_guide/classifier/OneRClassifier/index.html b/user_guide/classifier/OneRClassifier/index.html
index 9805759ed..550a6a584 100644
--- a/user_guide/classifier/OneRClassifier/index.html
+++ b/user_guide/classifier/OneRClassifier/index.html
@@ -12,25 +12,22 @@
         <link href="../../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../../cinder/css/base.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../../evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../../evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../../evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../../evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../../evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -952,14 +949,16 @@ <h3 id="methods">Methods</h3>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../../mathjaxhelper.js" defer></script>
-        <script src="../../../search/main.js" defer></script>
+        <script src="../../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../../mathjaxhelper.js"></script>
+        <script src="../../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/user_guide/classifier/Perceptron/index.html b/user_guide/classifier/Perceptron/index.html
index 892f6bec7..1b8569828 100644
--- a/user_guide/classifier/Perceptron/index.html
+++ b/user_guide/classifier/Perceptron/index.html
@@ -12,25 +12,22 @@
         <link href="../../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../../cinder/css/base.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../../evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../../evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../../evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../../evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../../evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -907,14 +904,16 @@ <h3 id="methods">Methods</h3>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../../mathjaxhelper.js" defer></script>
-        <script src="../../../search/main.js" defer></script>
+        <script src="../../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../../mathjaxhelper.js"></script>
+        <script src="../../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/user_guide/classifier/SoftmaxRegression/index.html b/user_guide/classifier/SoftmaxRegression/index.html
index aa218fd89..f3613febb 100644
--- a/user_guide/classifier/SoftmaxRegression/index.html
+++ b/user_guide/classifier/SoftmaxRegression/index.html
@@ -12,25 +12,22 @@
         <link href="../../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../../cinder/css/base.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../../evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../../evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../../evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../../evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../../evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -1095,14 +1092,16 @@ <h3 id="methods">Methods</h3>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../../mathjaxhelper.js" defer></script>
-        <script src="../../../search/main.js" defer></script>
+        <script src="../../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../../mathjaxhelper.js"></script>
+        <script src="../../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/user_guide/classifier/StackingCVClassifier/index.html b/user_guide/classifier/StackingCVClassifier/index.html
index 839cb461c..eef9bb652 100644
--- a/user_guide/classifier/StackingCVClassifier/index.html
+++ b/user_guide/classifier/StackingCVClassifier/index.html
@@ -12,25 +12,22 @@
         <link href="../../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../../cinder/css/base.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../../evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../../evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../../evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../../evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../../evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -1400,14 +1397,16 @@ <h3 id="properties">Properties</h3>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../../mathjaxhelper.js" defer></script>
-        <script src="../../../search/main.js" defer></script>
+        <script src="../../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../../mathjaxhelper.js"></script>
+        <script src="../../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/user_guide/classifier/StackingClassifier/index.html b/user_guide/classifier/StackingClassifier/index.html
index 9b1220b91..aac04dfe8 100644
--- a/user_guide/classifier/StackingClassifier/index.html
+++ b/user_guide/classifier/StackingClassifier/index.html
@@ -12,25 +12,22 @@
         <link href="../../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../../cinder/css/base.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../../evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../../evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../../evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../../evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../../evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -1099,14 +1096,16 @@ <h1 id="api">API</h1></div>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../../mathjaxhelper.js" defer></script>
-        <script src="../../../search/main.js" defer></script>
+        <script src="../../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../../mathjaxhelper.js"></script>
+        <script src="../../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/user_guide/cluster/Kmeans/index.html b/user_guide/cluster/Kmeans/index.html
index 1d4315c1f..065ca8af1 100644
--- a/user_guide/cluster/Kmeans/index.html
+++ b/user_guide/cluster/Kmeans/index.html
@@ -12,25 +12,22 @@
         <link href="../../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../../cinder/css/base.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../../evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../../evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../../evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../../evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../../evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -834,14 +831,16 @@ <h3 id="methods">Methods</h3>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../../mathjaxhelper.js" defer></script>
-        <script src="../../../search/main.js" defer></script>
+        <script src="../../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../../mathjaxhelper.js"></script>
+        <script src="../../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/user_guide/data/autompg_data/index.html b/user_guide/data/autompg_data/index.html
index dee4b863e..79e6ce72a 100644
--- a/user_guide/data/autompg_data/index.html
+++ b/user_guide/data/autompg_data/index.html
@@ -12,25 +12,22 @@
         <link href="../../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../../cinder/css/base.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../../evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../../evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../../evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../../evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../../evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -744,14 +741,16 @@ <h2 id="api">API</h2>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../../mathjaxhelper.js" defer></script>
-        <script src="../../../search/main.js" defer></script>
+        <script src="../../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../../mathjaxhelper.js"></script>
+        <script src="../../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/user_guide/data/boston_housing_data/index.html b/user_guide/data/boston_housing_data/index.html
index f6eb11025..859dfdb5a 100644
--- a/user_guide/data/boston_housing_data/index.html
+++ b/user_guide/data/boston_housing_data/index.html
@@ -12,25 +12,22 @@
         <link href="../../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../../cinder/css/base.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../../evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../../evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../../evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../../evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../../evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -745,14 +742,16 @@ <h2 id="api">API</h2>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../../mathjaxhelper.js" defer></script>
-        <script src="../../../search/main.js" defer></script>
+        <script src="../../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../../mathjaxhelper.js"></script>
+        <script src="../../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/user_guide/data/iris_data/index.html b/user_guide/data/iris_data/index.html
index 89319f8fa..25394d70a 100644
--- a/user_guide/data/iris_data/index.html
+++ b/user_guide/data/iris_data/index.html
@@ -12,25 +12,22 @@
         <link href="../../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../../cinder/css/base.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../../evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../../evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../../evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../../evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../../evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -756,14 +753,16 @@ <h2 id="api">API</h2>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../../mathjaxhelper.js" defer></script>
-        <script src="../../../search/main.js" defer></script>
+        <script src="../../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../../mathjaxhelper.js"></script>
+        <script src="../../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/user_guide/data/loadlocal_mnist/index.html b/user_guide/data/loadlocal_mnist/index.html
index b76d702af..a6e836726 100644
--- a/user_guide/data/loadlocal_mnist/index.html
+++ b/user_guide/data/loadlocal_mnist/index.html
@@ -12,25 +12,22 @@
         <link href="../../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../../cinder/css/base.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../../evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../../evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../../evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../../evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../../evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -803,14 +800,16 @@ <h2 id="api">API</h2>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../../mathjaxhelper.js" defer></script>
-        <script src="../../../search/main.js" defer></script>
+        <script src="../../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../../mathjaxhelper.js"></script>
+        <script src="../../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/user_guide/data/make_multiplexer_dataset/index.html b/user_guide/data/make_multiplexer_dataset/index.html
index a30570f9b..22db96e53 100644
--- a/user_guide/data/make_multiplexer_dataset/index.html
+++ b/user_guide/data/make_multiplexer_dataset/index.html
@@ -12,25 +12,22 @@
         <link href="../../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../../cinder/css/base.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../../evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../../evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../../evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../../evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../../evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -765,14 +762,16 @@ <h2 id="api">API</h2>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../../mathjaxhelper.js" defer></script>
-        <script src="../../../search/main.js" defer></script>
+        <script src="../../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../../mathjaxhelper.js"></script>
+        <script src="../../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/user_guide/data/mnist_data/index.html b/user_guide/data/mnist_data/index.html
index e5e465a66..26979d395 100644
--- a/user_guide/data/mnist_data/index.html
+++ b/user_guide/data/mnist_data/index.html
@@ -12,25 +12,22 @@
         <link href="../../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../../cinder/css/base.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../../evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../../evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../../evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../../evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../../evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -789,14 +786,16 @@ <h2 id="api">API</h2>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../../mathjaxhelper.js" defer></script>
-        <script src="../../../search/main.js" defer></script>
+        <script src="../../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../../mathjaxhelper.js"></script>
+        <script src="../../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/user_guide/data/three_blobs_data/index.html b/user_guide/data/three_blobs_data/index.html
index a5563f974..7d85320db 100644
--- a/user_guide/data/three_blobs_data/index.html
+++ b/user_guide/data/three_blobs_data/index.html
@@ -12,25 +12,22 @@
         <link href="../../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../../cinder/css/base.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../../evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../../evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../../evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../../evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../../evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -745,14 +742,16 @@ <h2 id="api">API</h2>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../../mathjaxhelper.js" defer></script>
-        <script src="../../../search/main.js" defer></script>
+        <script src="../../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../../mathjaxhelper.js"></script>
+        <script src="../../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/user_guide/data/wine_data/index.html b/user_guide/data/wine_data/index.html
index a03673669..dbad88252 100644
--- a/user_guide/data/wine_data/index.html
+++ b/user_guide/data/wine_data/index.html
@@ -12,25 +12,22 @@
         <link href="../../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../../cinder/css/base.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../../evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../../evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../../evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../../evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../../evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -860,14 +857,16 @@ <h2 id="api">API</h2>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../../mathjaxhelper.js" defer></script>
-        <script src="../../../search/main.js" defer></script>
+        <script src="../../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../../mathjaxhelper.js"></script>
+        <script src="../../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/user_guide/evaluate/BootstrapOutOfBag/index.html b/user_guide/evaluate/BootstrapOutOfBag/index.html
index 0f17e2b1d..d8f1a373a 100644
--- a/user_guide/evaluate/BootstrapOutOfBag/index.html
+++ b/user_guide/evaluate/BootstrapOutOfBag/index.html
@@ -12,25 +12,22 @@
         <link href="../../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../../cinder/css/base.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -802,14 +799,16 @@ <h3 id="methods">Methods</h3>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../../mathjaxhelper.js" defer></script>
-        <script src="../../../search/main.js" defer></script>
+        <script src="../../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../../mathjaxhelper.js"></script>
+        <script src="../../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/user_guide/evaluate/GroupTimeSeriesSplit/index.html b/user_guide/evaluate/GroupTimeSeriesSplit/index.html
index 08e7661d8..ded61dffa 100644
--- a/user_guide/evaluate/GroupTimeSeriesSplit/index.html
+++ b/user_guide/evaluate/GroupTimeSeriesSplit/index.html
@@ -12,25 +12,22 @@
         <link href="../../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../../cinder/css/base.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -1053,14 +1050,16 @@ <h3 id="methods">Methods</h3>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../../mathjaxhelper.js" defer></script>
-        <script src="../../../search/main.js" defer></script>
+        <script src="../../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../../mathjaxhelper.js"></script>
+        <script src="../../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/user_guide/evaluate/PredefinedHoldoutSplit/index.html b/user_guide/evaluate/PredefinedHoldoutSplit/index.html
index f183f7843..5df2ca5a7 100644
--- a/user_guide/evaluate/PredefinedHoldoutSplit/index.html
+++ b/user_guide/evaluate/PredefinedHoldoutSplit/index.html
@@ -12,25 +12,22 @@
         <link href="../../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../../cinder/css/base.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -782,14 +779,16 @@ <h3 id="methods">Methods</h3>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../../mathjaxhelper.js" defer></script>
-        <script src="../../../search/main.js" defer></script>
+        <script src="../../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../../mathjaxhelper.js"></script>
+        <script src="../../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/user_guide/evaluate/RandomHoldoutSplit/index.html b/user_guide/evaluate/RandomHoldoutSplit/index.html
index f7dfb9b8a..4fe4da73a 100644
--- a/user_guide/evaluate/RandomHoldoutSplit/index.html
+++ b/user_guide/evaluate/RandomHoldoutSplit/index.html
@@ -12,25 +12,22 @@
         <link href="../../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../../cinder/css/base.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -789,14 +786,16 @@ <h3 id="methods">Methods</h3>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../../mathjaxhelper.js" defer></script>
-        <script src="../../../search/main.js" defer></script>
+        <script src="../../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../../mathjaxhelper.js"></script>
+        <script src="../../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/user_guide/evaluate/accuracy_score/index.html b/user_guide/evaluate/accuracy_score/index.html
index 58db298f8..7679ac9d9 100644
--- a/user_guide/evaluate/accuracy_score/index.html
+++ b/user_guide/evaluate/accuracy_score/index.html
@@ -12,25 +12,22 @@
         <link href="../../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../../cinder/css/base.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -778,14 +775,16 @@ <h2 id="api">API</h2>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../../mathjaxhelper.js" defer></script>
-        <script src="../../../search/main.js" defer></script>
+        <script src="../../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../../mathjaxhelper.js"></script>
+        <script src="../../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/user_guide/evaluate/bias_variance_decomp/index.html b/user_guide/evaluate/bias_variance_decomp/index.html
index 2f32ea135..03e2e6284 100644
--- a/user_guide/evaluate/bias_variance_decomp/index.html
+++ b/user_guide/evaluate/bias_variance_decomp/index.html
@@ -12,25 +12,22 @@
         <link href="../../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../../cinder/css/base.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -1062,14 +1059,16 @@ <h2 id="api">API</h2>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../../mathjaxhelper.js" defer></script>
-        <script src="../../../search/main.js" defer></script>
+        <script src="../../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../../mathjaxhelper.js"></script>
+        <script src="../../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/user_guide/evaluate/bootstrap/index.html b/user_guide/evaluate/bootstrap/index.html
index 64d597665..13d00f1c1 100644
--- a/user_guide/evaluate/bootstrap/index.html
+++ b/user_guide/evaluate/bootstrap/index.html
@@ -12,25 +12,22 @@
         <link href="../../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../../cinder/css/base.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -788,14 +785,16 @@ <h2 id="api">API</h2>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../../mathjaxhelper.js" defer></script>
-        <script src="../../../search/main.js" defer></script>
+        <script src="../../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../../mathjaxhelper.js"></script>
+        <script src="../../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/user_guide/evaluate/bootstrap_point632_score/index.html b/user_guide/evaluate/bootstrap_point632_score/index.html
index 4d105bc31..bd4698ccf 100644
--- a/user_guide/evaluate/bootstrap_point632_score/index.html
+++ b/user_guide/evaluate/bootstrap_point632_score/index.html
@@ -12,25 +12,22 @@
         <link href="../../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../../cinder/css/base.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -891,14 +888,16 @@ <h2 id="api">API</h2>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../../mathjaxhelper.js" defer></script>
-        <script src="../../../search/main.js" defer></script>
+        <script src="../../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../../mathjaxhelper.js"></script>
+        <script src="../../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/user_guide/evaluate/cochrans_q/index.html b/user_guide/evaluate/cochrans_q/index.html
index 03164c204..bb950773e 100644
--- a/user_guide/evaluate/cochrans_q/index.html
+++ b/user_guide/evaluate/cochrans_q/index.html
@@ -12,25 +12,22 @@
         <link href="../../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../../cinder/css/base.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -894,14 +891,16 @@ <h2 id="api">API</h2>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../../mathjaxhelper.js" defer></script>
-        <script src="../../../search/main.js" defer></script>
+        <script src="../../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../../mathjaxhelper.js"></script>
+        <script src="../../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/user_guide/evaluate/combined_ftest_5x2cv/index.html b/user_guide/evaluate/combined_ftest_5x2cv/index.html
index eb7149077..be60c7ffc 100644
--- a/user_guide/evaluate/combined_ftest_5x2cv/index.html
+++ b/user_guide/evaluate/combined_ftest_5x2cv/index.html
@@ -8,29 +8,26 @@
         <meta name="author" content="Sebastian Raschka">
         <link rel="canonical" href="https://rasbt.github.io/mlxtend/user_guide/evaluate/combined_ftest_5x2cv/">
         <link rel="shortcut icon" href="../../../img/favicon.ico">
-        <title>combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons - mlxtend</title>
+        <title>combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons - mlxtend</title>
         <link href="../../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../../cinder/css/base.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="./" class="dropdown-item active">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="./" class="dropdown-item active">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -807,14 +804,16 @@ <h2 id="api">API</h2>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../../mathjaxhelper.js" defer></script>
-        <script src="../../../search/main.js" defer></script>
+        <script src="../../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../../mathjaxhelper.js"></script>
+        <script src="../../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/user_guide/evaluate/confusion_matrix/index.html b/user_guide/evaluate/confusion_matrix/index.html
index 69559ebab..21a7c72b6 100644
--- a/user_guide/evaluate/confusion_matrix/index.html
+++ b/user_guide/evaluate/confusion_matrix/index.html
@@ -12,25 +12,22 @@
         <link href="../../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../../cinder/css/base.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -773,14 +770,16 @@ <h2 id="api">API</h2>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../../mathjaxhelper.js" defer></script>
-        <script src="../../../search/main.js" defer></script>
+        <script src="../../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../../mathjaxhelper.js"></script>
+        <script src="../../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/user_guide/evaluate/create_counterfactual/index.html b/user_guide/evaluate/create_counterfactual/index.html
index 2a4dcf8be..a01111479 100644
--- a/user_guide/evaluate/create_counterfactual/index.html
+++ b/user_guide/evaluate/create_counterfactual/index.html
@@ -12,25 +12,22 @@
         <link href="../../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../../cinder/css/base.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -935,14 +932,16 @@ <h2 id="api">API</h2>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../../mathjaxhelper.js" defer></script>
-        <script src="../../../search/main.js" defer></script>
+        <script src="../../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../../mathjaxhelper.js"></script>
+        <script src="../../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/user_guide/evaluate/feature_importance_permutation/index.html b/user_guide/evaluate/feature_importance_permutation/index.html
index 39eb62af0..2f36ff523 100644
--- a/user_guide/evaluate/feature_importance_permutation/index.html
+++ b/user_guide/evaluate/feature_importance_permutation/index.html
@@ -12,25 +12,22 @@
         <link href="../../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../../cinder/css/base.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -1144,14 +1141,16 @@ <h2 id="api">API</h2>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../../mathjaxhelper.js" defer></script>
-        <script src="../../../search/main.js" defer></script>
+        <script src="../../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../../mathjaxhelper.js"></script>
+        <script src="../../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/user_guide/evaluate/ftest/index.html b/user_guide/evaluate/ftest/index.html
index 4d0dfb598..0306224e3 100644
--- a/user_guide/evaluate/ftest/index.html
+++ b/user_guide/evaluate/ftest/index.html
@@ -12,25 +12,22 @@
         <link href="../../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../../cinder/css/base.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -794,14 +791,16 @@ <h2 id="api">API</h2>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../../mathjaxhelper.js" defer></script>
-        <script src="../../../search/main.js" defer></script>
+        <script src="../../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../../mathjaxhelper.js"></script>
+        <script src="../../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/user_guide/evaluate/lift_score/index.html b/user_guide/evaluate/lift_score/index.html
index 869173bbb..66b46167c 100644
--- a/user_guide/evaluate/lift_score/index.html
+++ b/user_guide/evaluate/lift_score/index.html
@@ -12,25 +12,22 @@
         <link href="../../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../../cinder/css/base.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -785,14 +782,16 @@ <h2 id="api">API</h2>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../../mathjaxhelper.js" defer></script>
-        <script src="../../../search/main.js" defer></script>
+        <script src="../../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../../mathjaxhelper.js"></script>
+        <script src="../../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/user_guide/evaluate/mcnemar/index.html b/user_guide/evaluate/mcnemar/index.html
index ad24997fb..0697d884c 100644
--- a/user_guide/evaluate/mcnemar/index.html
+++ b/user_guide/evaluate/mcnemar/index.html
@@ -12,25 +12,22 @@
         <link href="../../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../../cinder/css/base.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -802,14 +799,16 @@ <h2 id="api">API</h2>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../../mathjaxhelper.js" defer></script>
-        <script src="../../../search/main.js" defer></script>
+        <script src="../../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../../mathjaxhelper.js"></script>
+        <script src="../../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/user_guide/evaluate/mcnemar_table/index.html b/user_guide/evaluate/mcnemar_table/index.html
index 3ddf5b1e7..462a5bf96 100644
--- a/user_guide/evaluate/mcnemar_table/index.html
+++ b/user_guide/evaluate/mcnemar_table/index.html
@@ -12,25 +12,22 @@
         <link href="../../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../../cinder/css/base.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -734,14 +731,16 @@ <h2 id="api">API</h2>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../../mathjaxhelper.js" defer></script>
-        <script src="../../../search/main.js" defer></script>
+        <script src="../../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../../mathjaxhelper.js"></script>
+        <script src="../../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/user_guide/evaluate/mcnemar_tables/index.html b/user_guide/evaluate/mcnemar_tables/index.html
index 51cd445f6..8d51fbed8 100644
--- a/user_guide/evaluate/mcnemar_tables/index.html
+++ b/user_guide/evaluate/mcnemar_tables/index.html
@@ -12,25 +12,22 @@
         <link href="../../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../../cinder/css/base.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -790,14 +787,16 @@ <h2 id="api">API</h2>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../../mathjaxhelper.js" defer></script>
-        <script src="../../../search/main.js" defer></script>
+        <script src="../../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../../mathjaxhelper.js"></script>
+        <script src="../../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/user_guide/evaluate/paired_ttest_5x2cv/index.html b/user_guide/evaluate/paired_ttest_5x2cv/index.html
index 2587d48d4..44fa3afd3 100644
--- a/user_guide/evaluate/paired_ttest_5x2cv/index.html
+++ b/user_guide/evaluate/paired_ttest_5x2cv/index.html
@@ -8,29 +8,26 @@
         <meta name="author" content="Sebastian Raschka">
         <link rel="canonical" href="https://rasbt.github.io/mlxtend/user_guide/evaluate/paired_ttest_5x2cv/">
         <link rel="shortcut icon" href="../../../img/favicon.ico">
-        <title>paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons - mlxtend</title>
+        <title>paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons - mlxtend</title>
         <link href="../../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../../cinder/css/base.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="./" class="dropdown-item active">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="./" class="dropdown-item active">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -803,14 +800,16 @@ <h2 id="api">API</h2>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../../mathjaxhelper.js" defer></script>
-        <script src="../../../search/main.js" defer></script>
+        <script src="../../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../../mathjaxhelper.js"></script>
+        <script src="../../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/user_guide/evaluate/paired_ttest_kfold_cv/index.html b/user_guide/evaluate/paired_ttest_kfold_cv/index.html
index 1a09d0431..0a0885472 100644
--- a/user_guide/evaluate/paired_ttest_kfold_cv/index.html
+++ b/user_guide/evaluate/paired_ttest_kfold_cv/index.html
@@ -8,29 +8,26 @@
         <meta name="author" content="Sebastian Raschka">
         <link rel="canonical" href="https://rasbt.github.io/mlxtend/user_guide/evaluate/paired_ttest_kfold_cv/">
         <link rel="shortcut icon" href="../../../img/favicon.ico">
-        <title>paired_ttest_kfold_cv: K-fold cross-validated paired *t* test - mlxtend</title>
+        <title>paired_ttest_kfold_cv: K-fold cross-validated paired t test - mlxtend</title>
         <link href="../../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../../cinder/css/base.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="./" class="dropdown-item active">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="./" class="dropdown-item active">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -803,14 +800,16 @@ <h2 id="api">API</h2>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../../mathjaxhelper.js" defer></script>
-        <script src="../../../search/main.js" defer></script>
+        <script src="../../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../../mathjaxhelper.js"></script>
+        <script src="../../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/user_guide/evaluate/paired_ttest_resampled/index.html b/user_guide/evaluate/paired_ttest_resampled/index.html
index 16ee75b37..b6f192f9e 100644
--- a/user_guide/evaluate/paired_ttest_resampled/index.html
+++ b/user_guide/evaluate/paired_ttest_resampled/index.html
@@ -8,29 +8,26 @@
         <meta name="author" content="Sebastian Raschka">
         <link rel="canonical" href="https://rasbt.github.io/mlxtend/user_guide/evaluate/paired_ttest_resampled/">
         <link rel="shortcut icon" href="../../../img/favicon.ico">
-        <title>paired_ttest_resample: Resampled paired *t* test - mlxtend</title>
+        <title>paired_ttest_resample: Resampled paired t test - mlxtend</title>
         <link href="../../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../../cinder/css/base.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="./" class="dropdown-item active">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="./" class="dropdown-item active">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -816,14 +813,16 @@ <h2 id="api">API</h2>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../../mathjaxhelper.js" defer></script>
-        <script src="../../../search/main.js" defer></script>
+        <script src="../../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../../mathjaxhelper.js"></script>
+        <script src="../../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/user_guide/evaluate/permutation_test/index.html b/user_guide/evaluate/permutation_test/index.html
index 8b1e657f5..3b731f9b3 100644
--- a/user_guide/evaluate/permutation_test/index.html
+++ b/user_guide/evaluate/permutation_test/index.html
@@ -12,25 +12,22 @@
         <link href="../../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../../cinder/css/base.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -817,14 +814,16 @@ <h2 id="api">API</h2>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../../mathjaxhelper.js" defer></script>
-        <script src="../../../search/main.js" defer></script>
+        <script src="../../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../../mathjaxhelper.js"></script>
+        <script src="../../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/user_guide/evaluate/proportion_difference/index.html b/user_guide/evaluate/proportion_difference/index.html
index 1b975190a..6ecf9327a 100644
--- a/user_guide/evaluate/proportion_difference/index.html
+++ b/user_guide/evaluate/proportion_difference/index.html
@@ -12,25 +12,22 @@
         <link href="../../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../../cinder/css/base.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -788,14 +785,16 @@ <h2 id="api">API</h2>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../../mathjaxhelper.js" defer></script>
-        <script src="../../../search/main.js" defer></script>
+        <script src="../../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../../mathjaxhelper.js"></script>
+        <script src="../../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/user_guide/evaluate/scoring/index.html b/user_guide/evaluate/scoring/index.html
index 21ed0b415..b28e0b2db 100644
--- a/user_guide/evaluate/scoring/index.html
+++ b/user_guide/evaluate/scoring/index.html
@@ -12,25 +12,22 @@
         <link href="../../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../../cinder/css/base.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -792,14 +789,16 @@ <h2 id="api">API</h2>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../../mathjaxhelper.js" defer></script>
-        <script src="../../../search/main.js" defer></script>
+        <script src="../../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../../mathjaxhelper.js"></script>
+        <script src="../../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/user_guide/feature_extraction/LinearDiscriminantAnalysis/index.html b/user_guide/feature_extraction/LinearDiscriminantAnalysis/index.html
index 8f29acbf2..f81f51f07 100644
--- a/user_guide/feature_extraction/LinearDiscriminantAnalysis/index.html
+++ b/user_guide/feature_extraction/LinearDiscriminantAnalysis/index.html
@@ -12,25 +12,22 @@
         <link href="../../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../../cinder/css/base.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../../evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../../evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../../evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../../evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../../evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -815,14 +812,16 @@ <h3 id="methods">Methods</h3>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../../mathjaxhelper.js" defer></script>
-        <script src="../../../search/main.js" defer></script>
+        <script src="../../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../../mathjaxhelper.js"></script>
+        <script src="../../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/user_guide/feature_extraction/PrincipalComponentAnalysis/index.html b/user_guide/feature_extraction/PrincipalComponentAnalysis/index.html
index a861e8342..4ae870fb5 100644
--- a/user_guide/feature_extraction/PrincipalComponentAnalysis/index.html
+++ b/user_guide/feature_extraction/PrincipalComponentAnalysis/index.html
@@ -12,25 +12,22 @@
         <link href="../../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../../cinder/css/base.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../../evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../../evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../../evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../../evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../../evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -1080,14 +1077,16 @@ <h3 id="methods">Methods</h3>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../../mathjaxhelper.js" defer></script>
-        <script src="../../../search/main.js" defer></script>
+        <script src="../../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../../mathjaxhelper.js"></script>
+        <script src="../../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/user_guide/feature_extraction/RBFKernelPCA/index.html b/user_guide/feature_extraction/RBFKernelPCA/index.html
index 35e8c0ca9..665beaa6b 100644
--- a/user_guide/feature_extraction/RBFKernelPCA/index.html
+++ b/user_guide/feature_extraction/RBFKernelPCA/index.html
@@ -12,25 +12,22 @@
         <link href="../../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../../cinder/css/base.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../../evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../../evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../../evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../../evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../../evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -959,14 +956,16 @@ <h3 id="methods">Methods</h3>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../../mathjaxhelper.js" defer></script>
-        <script src="../../../search/main.js" defer></script>
+        <script src="../../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../../mathjaxhelper.js"></script>
+        <script src="../../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/user_guide/feature_selection/ColumnSelector/index.html b/user_guide/feature_selection/ColumnSelector/index.html
index 95882f895..9903546b6 100644
--- a/user_guide/feature_selection/ColumnSelector/index.html
+++ b/user_guide/feature_selection/ColumnSelector/index.html
@@ -12,25 +12,22 @@
         <link href="../../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../../cinder/css/base.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../../evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../../evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../../evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../../evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../../evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -968,14 +965,16 @@ <h3 id="methods">Methods</h3>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../../mathjaxhelper.js" defer></script>
-        <script src="../../../search/main.js" defer></script>
+        <script src="../../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../../mathjaxhelper.js"></script>
+        <script src="../../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/user_guide/feature_selection/ExhaustiveFeatureSelector/index.html b/user_guide/feature_selection/ExhaustiveFeatureSelector/index.html
index 3a49d9844..f58b061e9 100644
--- a/user_guide/feature_selection/ExhaustiveFeatureSelector/index.html
+++ b/user_guide/feature_selection/ExhaustiveFeatureSelector/index.html
@@ -12,25 +12,22 @@
         <link href="../../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../../cinder/css/base.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../../evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../../evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../../evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../../evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../../evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -1925,14 +1922,16 @@ <h3 id="methods">Methods</h3>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../../mathjaxhelper.js" defer></script>
-        <script src="../../../search/main.js" defer></script>
+        <script src="../../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../../mathjaxhelper.js"></script>
+        <script src="../../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/user_guide/feature_selection/SequentialFeatureSelector/index.html b/user_guide/feature_selection/SequentialFeatureSelector/index.html
index 7b34e9a96..daeb620da 100644
--- a/user_guide/feature_selection/SequentialFeatureSelector/index.html
+++ b/user_guide/feature_selection/SequentialFeatureSelector/index.html
@@ -12,25 +12,22 @@
         <link href="../../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../../cinder/css/base.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../../evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../../evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../../evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../../evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../../evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -2528,14 +2525,16 @@ <h3 id="properties">Properties</h3>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../../mathjaxhelper.js" defer></script>
-        <script src="../../../search/main.js" defer></script>
+        <script src="../../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../../mathjaxhelper.js"></script>
+        <script src="../../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/user_guide/file_io/find_filegroups/index.html b/user_guide/file_io/find_filegroups/index.html
index 319fe9813..6598e29d2 100644
--- a/user_guide/file_io/find_filegroups/index.html
+++ b/user_guide/file_io/find_filegroups/index.html
@@ -12,25 +12,22 @@
         <link href="../../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../../cinder/css/base.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../../evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../../evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../../evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../../evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../../evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -761,14 +758,16 @@ <h2 id="api">API</h2>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../../mathjaxhelper.js" defer></script>
-        <script src="../../../search/main.js" defer></script>
+        <script src="../../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../../mathjaxhelper.js"></script>
+        <script src="../../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/user_guide/file_io/find_files/index.html b/user_guide/file_io/find_files/index.html
index 49e44ff37..c7220ecfc 100644
--- a/user_guide/file_io/find_files/index.html
+++ b/user_guide/file_io/find_files/index.html
@@ -12,25 +12,22 @@
         <link href="../../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../../cinder/css/base.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../../evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../../evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../../evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../../evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../../evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -726,14 +723,16 @@ <h2 id="api">API</h2>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../../mathjaxhelper.js" defer></script>
-        <script src="../../../search/main.js" defer></script>
+        <script src="../../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../../mathjaxhelper.js"></script>
+        <script src="../../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/user_guide/frequent_patterns/apriori/index.html b/user_guide/frequent_patterns/apriori/index.html
index d037fd4df..51cb08e7a 100644
--- a/user_guide/frequent_patterns/apriori/index.html
+++ b/user_guide/frequent_patterns/apriori/index.html
@@ -12,25 +12,22 @@
         <link href="../../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../../cinder/css/base.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../../evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../../evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../../evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../../evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../../evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -1437,14 +1434,16 @@ <h2 id="api">API</h2>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../../mathjaxhelper.js" defer></script>
-        <script src="../../../search/main.js" defer></script>
+        <script src="../../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../../mathjaxhelper.js"></script>
+        <script src="../../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/user_guide/frequent_patterns/association_rules/index.html b/user_guide/frequent_patterns/association_rules/index.html
index af1c27ee8..4376b6b65 100644
--- a/user_guide/frequent_patterns/association_rules/index.html
+++ b/user_guide/frequent_patterns/association_rules/index.html
@@ -12,25 +12,22 @@
         <link href="../../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../../cinder/css/base.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../../evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../../evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../../evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../../evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../../evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -2149,14 +2146,16 @@ <h2 id="api">API</h2></div>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../../mathjaxhelper.js" defer></script>
-        <script src="../../../search/main.js" defer></script>
+        <script src="../../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../../mathjaxhelper.js"></script>
+        <script src="../../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/user_guide/frequent_patterns/fpgrowth/index.html b/user_guide/frequent_patterns/fpgrowth/index.html
index ab24fb2d5..2418bcd58 100644
--- a/user_guide/frequent_patterns/fpgrowth/index.html
+++ b/user_guide/frequent_patterns/fpgrowth/index.html
@@ -12,25 +12,22 @@
         <link href="../../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../../cinder/css/base.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../../evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../../evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../../evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../../evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../../evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -1069,14 +1066,16 @@ <h2 id="api">API</h2>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../../mathjaxhelper.js" defer></script>
-        <script src="../../../search/main.js" defer></script>
+        <script src="../../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../../mathjaxhelper.js"></script>
+        <script src="../../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/user_guide/frequent_patterns/fpmax/index.html b/user_guide/frequent_patterns/fpmax/index.html
index 3f508497e..eae8cc504 100644
--- a/user_guide/frequent_patterns/fpmax/index.html
+++ b/user_guide/frequent_patterns/fpmax/index.html
@@ -12,25 +12,22 @@
         <link href="../../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../../cinder/css/base.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../../evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../../evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../../evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../../evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../../evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -966,14 +963,16 @@ <h2 id="api">API</h2>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../../mathjaxhelper.js" defer></script>
-        <script src="../../../search/main.js" defer></script>
+        <script src="../../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../../mathjaxhelper.js"></script>
+        <script src="../../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/user_guide/frequent_patterns/hmine/index.html b/user_guide/frequent_patterns/hmine/index.html
index 6e48d2c99..17ba0f354 100644
--- a/user_guide/frequent_patterns/hmine/index.html
+++ b/user_guide/frequent_patterns/hmine/index.html
@@ -12,25 +12,22 @@
         <link href="../../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../../cinder/css/base.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../../evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../../evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../../evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../../evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../../evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -1282,14 +1279,16 @@ <h2 id="api">API</h2>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../../mathjaxhelper.js" defer></script>
-        <script src="../../../search/main.js" defer></script>
+        <script src="../../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../../mathjaxhelper.js"></script>
+        <script src="../../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/user_guide/image/extract_face_landmarks/index.html b/user_guide/image/extract_face_landmarks/index.html
index 1e905948f..39b1532b7 100644
--- a/user_guide/image/extract_face_landmarks/index.html
+++ b/user_guide/image/extract_face_landmarks/index.html
@@ -12,25 +12,22 @@
         <link href="../../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../../cinder/css/base.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../../evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../../evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../../evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../../evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../../evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -833,14 +830,16 @@ <h2 id="api">API</h2>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../../mathjaxhelper.js" defer></script>
-        <script src="../../../search/main.js" defer></script>
+        <script src="../../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../../mathjaxhelper.js"></script>
+        <script src="../../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/user_guide/image/eyepad_align/index.html b/user_guide/image/eyepad_align/index.html
index bdf6cc839..0f11c215b 100644
--- a/user_guide/image/eyepad_align/index.html
+++ b/user_guide/image/eyepad_align/index.html
@@ -12,25 +12,22 @@
         <link href="../../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../../cinder/css/base.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../../evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../../evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../../evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../../evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../../evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -1063,14 +1060,16 @@ <h3 id="methods">Methods</h3>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../../mathjaxhelper.js" defer></script>
-        <script src="../../../search/main.js" defer></script>
+        <script src="../../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../../mathjaxhelper.js"></script>
+        <script src="../../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/user_guide/math/num_combinations/index.html b/user_guide/math/num_combinations/index.html
index cd9eedc11..949eee7ea 100644
--- a/user_guide/math/num_combinations/index.html
+++ b/user_guide/math/num_combinations/index.html
@@ -8,29 +8,26 @@
         <meta name="author" content="Sebastian Raschka">
         <link rel="canonical" href="https://rasbt.github.io/mlxtend/user_guide/math/num_combinations/">
         <link rel="shortcut icon" href="../../../img/favicon.ico">
-        <title>num_combinations: combinations for creating subsequences of *k* elements - mlxtend</title>
+        <title>num_combinations: combinations for creating subsequences of k elements - mlxtend</title>
         <link href="../../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../../cinder/css/base.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../../evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../../evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../../evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../../evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../../evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="./" class="dropdown-item active">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="./" class="dropdown-item active">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -757,14 +754,16 @@ <h2 id="api">API</h2>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../../mathjaxhelper.js" defer></script>
-        <script src="../../../search/main.js" defer></script>
+        <script src="../../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../../mathjaxhelper.js"></script>
+        <script src="../../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/user_guide/math/num_permutations/index.html b/user_guide/math/num_permutations/index.html
index 8c993b612..cbd9ae9a3 100644
--- a/user_guide/math/num_permutations/index.html
+++ b/user_guide/math/num_permutations/index.html
@@ -8,29 +8,26 @@
         <meta name="author" content="Sebastian Raschka">
         <link rel="canonical" href="https://rasbt.github.io/mlxtend/user_guide/math/num_permutations/">
         <link rel="shortcut icon" href="../../../img/favicon.ico">
-        <title>num_permutations: number of permutations for creating subsequences of *k* elements - mlxtend</title>
+        <title>num_permutations: number of permutations for creating subsequences of k elements - mlxtend</title>
         <link href="../../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../../cinder/css/base.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../../evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../../evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../../evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../../evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../../evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="./" class="dropdown-item active">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="./" class="dropdown-item active">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -747,14 +744,16 @@ <h2 id="api">API</h2>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../../mathjaxhelper.js" defer></script>
-        <script src="../../../search/main.js" defer></script>
+        <script src="../../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../../mathjaxhelper.js"></script>
+        <script src="../../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/user_guide/math/vectorspace_dimensionality/index.html b/user_guide/math/vectorspace_dimensionality/index.html
index 8bb45f5f9..4d6bea472 100644
--- a/user_guide/math/vectorspace_dimensionality/index.html
+++ b/user_guide/math/vectorspace_dimensionality/index.html
@@ -12,25 +12,22 @@
         <link href="../../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../../cinder/css/base.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../../evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../../evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../../evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../../evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../../evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -720,14 +717,16 @@ <h2 id="api">API</h2>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../../mathjaxhelper.js" defer></script>
-        <script src="../../../search/main.js" defer></script>
+        <script src="../../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../../mathjaxhelper.js"></script>
+        <script src="../../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/user_guide/math/vectorspace_orthonormalization/index.html b/user_guide/math/vectorspace_orthonormalization/index.html
index d31600e62..8cacc03a9 100644
--- a/user_guide/math/vectorspace_orthonormalization/index.html
+++ b/user_guide/math/vectorspace_orthonormalization/index.html
@@ -12,25 +12,22 @@
         <link href="../../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../../cinder/css/base.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../../evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../../evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../../evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../../evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../../evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -735,14 +732,16 @@ <h2 id="api">API</h2>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../../mathjaxhelper.js" defer></script>
-        <script src="../../../search/main.js" defer></script>
+        <script src="../../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../../mathjaxhelper.js"></script>
+        <script src="../../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/user_guide/plotting/category_scatter/index.html b/user_guide/plotting/category_scatter/index.html
index 4033c0302..c4ec9e4a0 100644
--- a/user_guide/plotting/category_scatter/index.html
+++ b/user_guide/plotting/category_scatter/index.html
@@ -12,25 +12,22 @@
         <link href="../../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../../cinder/css/base.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../../evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../../evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../../evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../../evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../../evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -814,14 +811,16 @@ <h2 id="api">API</h2>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../../mathjaxhelper.js" defer></script>
-        <script src="../../../search/main.js" defer></script>
+        <script src="../../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../../mathjaxhelper.js"></script>
+        <script src="../../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/user_guide/plotting/checkerboard_plot/index.html b/user_guide/plotting/checkerboard_plot/index.html
index bbf41b48c..3d122e682 100644
--- a/user_guide/plotting/checkerboard_plot/index.html
+++ b/user_guide/plotting/checkerboard_plot/index.html
@@ -12,25 +12,22 @@
         <link href="../../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../../cinder/css/base.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../../evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../../evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../../evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../../evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../../evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -740,14 +737,16 @@ <h2 id="api">API</h2>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../../mathjaxhelper.js" defer></script>
-        <script src="../../../search/main.js" defer></script>
+        <script src="../../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../../mathjaxhelper.js"></script>
+        <script src="../../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/user_guide/plotting/ecdf/index.html b/user_guide/plotting/ecdf/index.html
index d7b32527f..a20cbaf48 100644
--- a/user_guide/plotting/ecdf/index.html
+++ b/user_guide/plotting/ecdf/index.html
@@ -12,25 +12,22 @@
         <link href="../../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../../cinder/css/base.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../../evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../../evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../../evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../../evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../../evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -782,14 +779,16 @@ <h2 id="api">API</h2>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../../mathjaxhelper.js" defer></script>
-        <script src="../../../search/main.js" defer></script>
+        <script src="../../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../../mathjaxhelper.js"></script>
+        <script src="../../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/user_guide/plotting/enrichment_plot/index.html b/user_guide/plotting/enrichment_plot/index.html
index c67cf7a2a..5e402676b 100644
--- a/user_guide/plotting/enrichment_plot/index.html
+++ b/user_guide/plotting/enrichment_plot/index.html
@@ -12,25 +12,22 @@
         <link href="../../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../../cinder/css/base.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../../evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../../evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../../evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../../evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../../evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -785,14 +782,16 @@ <h2 id="api">API</h2>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../../mathjaxhelper.js" defer></script>
-        <script src="../../../search/main.js" defer></script>
+        <script src="../../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../../mathjaxhelper.js"></script>
+        <script src="../../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/user_guide/plotting/heatmap/index.html b/user_guide/plotting/heatmap/index.html
index 97c326244..6d83eae02 100644
--- a/user_guide/plotting/heatmap/index.html
+++ b/user_guide/plotting/heatmap/index.html
@@ -12,25 +12,22 @@
         <link href="../../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../../cinder/css/base.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../../evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../../evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../../evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../../evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../../evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -906,14 +903,16 @@ <h2 id="api">API</h2>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../../mathjaxhelper.js" defer></script>
-        <script src="../../../search/main.js" defer></script>
+        <script src="../../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../../mathjaxhelper.js"></script>
+        <script src="../../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/user_guide/plotting/plot_confusion_matrix/index.html b/user_guide/plotting/plot_confusion_matrix/index.html
index 5c894f03c..c216fb47d 100644
--- a/user_guide/plotting/plot_confusion_matrix/index.html
+++ b/user_guide/plotting/plot_confusion_matrix/index.html
@@ -12,25 +12,22 @@
         <link href="../../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../../cinder/css/base.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../../evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../../evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../../evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../../evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../../evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -874,14 +871,16 @@ <h2 id="api">API</h2>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../../mathjaxhelper.js" defer></script>
-        <script src="../../../search/main.js" defer></script>
+        <script src="../../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../../mathjaxhelper.js"></script>
+        <script src="../../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/user_guide/plotting/plot_decision_regions/index.html b/user_guide/plotting/plot_decision_regions/index.html
index b8d25003a..15c1c96ea 100644
--- a/user_guide/plotting/plot_decision_regions/index.html
+++ b/user_guide/plotting/plot_decision_regions/index.html
@@ -12,25 +12,22 @@
         <link href="../../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../../cinder/css/base.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../../evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../../evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../../evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../../evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../../evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -1311,14 +1308,16 @@ <h1 id="api">API</h1>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../../mathjaxhelper.js" defer></script>
-        <script src="../../../search/main.js" defer></script>
+        <script src="../../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../../mathjaxhelper.js"></script>
+        <script src="../../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/user_guide/plotting/plot_learning_curves/index.html b/user_guide/plotting/plot_learning_curves/index.html
index 0fadab9ac..3f4116253 100644
--- a/user_guide/plotting/plot_learning_curves/index.html
+++ b/user_guide/plotting/plot_learning_curves/index.html
@@ -12,25 +12,22 @@
         <link href="../../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../../cinder/css/base.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../../evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../../evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../../evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../../evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../../evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -753,14 +750,16 @@ <h2 id="api">API</h2>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../../mathjaxhelper.js" defer></script>
-        <script src="../../../search/main.js" defer></script>
+        <script src="../../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../../mathjaxhelper.js"></script>
+        <script src="../../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/user_guide/plotting/plot_linear_regression/index.html b/user_guide/plotting/plot_linear_regression/index.html
index 64f77a97f..9f45eb1ea 100644
--- a/user_guide/plotting/plot_linear_regression/index.html
+++ b/user_guide/plotting/plot_linear_regression/index.html
@@ -12,25 +12,22 @@
         <link href="../../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../../cinder/css/base.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../../evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../../evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../../evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../../evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../../evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -717,14 +714,16 @@ <h2 id="api">API</h2>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../../mathjaxhelper.js" defer></script>
-        <script src="../../../search/main.js" defer></script>
+        <script src="../../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../../mathjaxhelper.js"></script>
+        <script src="../../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/user_guide/plotting/plot_pca_correlation_graph/index.html b/user_guide/plotting/plot_pca_correlation_graph/index.html
index 303eb2fab..6cb891f90 100644
--- a/user_guide/plotting/plot_pca_correlation_graph/index.html
+++ b/user_guide/plotting/plot_pca_correlation_graph/index.html
@@ -12,25 +12,22 @@
         <link href="../../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../../cinder/css/base.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../../evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../../evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../../evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../../evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../../evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -771,14 +768,16 @@ <h2 id="api">API</h2>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../../mathjaxhelper.js" defer></script>
-        <script src="../../../search/main.js" defer></script>
+        <script src="../../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../../mathjaxhelper.js"></script>
+        <script src="../../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/user_guide/plotting/plot_sequential_feature_selection/index.html b/user_guide/plotting/plot_sequential_feature_selection/index.html
index 0d9b87648..6a2dd840f 100644
--- a/user_guide/plotting/plot_sequential_feature_selection/index.html
+++ b/user_guide/plotting/plot_sequential_feature_selection/index.html
@@ -12,25 +12,22 @@
         <link href="../../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../../cinder/css/base.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../../evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../../evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../../evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../../evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../../evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -741,14 +738,16 @@ <h1 id="api">API</h1>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../../mathjaxhelper.js" defer></script>
-        <script src="../../../search/main.js" defer></script>
+        <script src="../../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../../mathjaxhelper.js"></script>
+        <script src="../../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/user_guide/plotting/scatter_hist/index.html b/user_guide/plotting/scatter_hist/index.html
index 83de8675f..5549ba310 100644
--- a/user_guide/plotting/scatter_hist/index.html
+++ b/user_guide/plotting/scatter_hist/index.html
@@ -12,25 +12,22 @@
         <link href="../../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../../cinder/css/base.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../../evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../../evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../../evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../../evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../../evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -796,14 +793,16 @@ <h1 id="api">API</h1>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../../mathjaxhelper.js" defer></script>
-        <script src="../../../search/main.js" defer></script>
+        <script src="../../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../../mathjaxhelper.js"></script>
+        <script src="../../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/user_guide/plotting/scatterplotmatrix/index.html b/user_guide/plotting/scatterplotmatrix/index.html
index cfbef0d17..2e29f7aff 100644
--- a/user_guide/plotting/scatterplotmatrix/index.html
+++ b/user_guide/plotting/scatterplotmatrix/index.html
@@ -12,25 +12,22 @@
         <link href="../../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../../cinder/css/base.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../../evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../../evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../../evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../../evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../../evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -735,14 +732,16 @@ <h2 id="api">API</h2>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../../mathjaxhelper.js" defer></script>
-        <script src="../../../search/main.js" defer></script>
+        <script src="../../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../../mathjaxhelper.js"></script>
+        <script src="../../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/user_guide/plotting/stacked_barplot/index.html b/user_guide/plotting/stacked_barplot/index.html
index e2b2bc634..dbabca0de 100644
--- a/user_guide/plotting/stacked_barplot/index.html
+++ b/user_guide/plotting/stacked_barplot/index.html
@@ -12,25 +12,22 @@
         <link href="../../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../../cinder/css/base.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../../evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../../evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../../evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../../evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../../evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -761,14 +758,16 @@ <h2 id="api">API</h2>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../../mathjaxhelper.js" defer></script>
-        <script src="../../../search/main.js" defer></script>
+        <script src="../../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../../mathjaxhelper.js"></script>
+        <script src="../../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/user_guide/preprocessing/CopyTransformer/index.html b/user_guide/preprocessing/CopyTransformer/index.html
index 6d8963c88..b5ffdf976 100644
--- a/user_guide/preprocessing/CopyTransformer/index.html
+++ b/user_guide/preprocessing/CopyTransformer/index.html
@@ -12,25 +12,22 @@
         <link href="../../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../../cinder/css/base.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../../evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../../evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../../evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../../evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../../evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -793,14 +790,16 @@ <h3 id="methods">Methods</h3>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../../mathjaxhelper.js" defer></script>
-        <script src="../../../search/main.js" defer></script>
+        <script src="../../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../../mathjaxhelper.js"></script>
+        <script src="../../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/user_guide/preprocessing/DenseTransformer/index.html b/user_guide/preprocessing/DenseTransformer/index.html
index 82b066c45..1616dc22a 100644
--- a/user_guide/preprocessing/DenseTransformer/index.html
+++ b/user_guide/preprocessing/DenseTransformer/index.html
@@ -12,25 +12,22 @@
         <link href="../../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../../cinder/css/base.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../../evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../../evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../../evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../../evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../../evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -793,14 +790,16 @@ <h3 id="methods">Methods</h3>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../../mathjaxhelper.js" defer></script>
-        <script src="../../../search/main.js" defer></script>
+        <script src="../../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../../mathjaxhelper.js"></script>
+        <script src="../../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/user_guide/preprocessing/MeanCenterer/index.html b/user_guide/preprocessing/MeanCenterer/index.html
index 7523d868f..a0165efe0 100644
--- a/user_guide/preprocessing/MeanCenterer/index.html
+++ b/user_guide/preprocessing/MeanCenterer/index.html
@@ -12,25 +12,22 @@
         <link href="../../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../../cinder/css/base.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../../evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../../evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../../evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../../evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../../evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -733,14 +730,16 @@ <h3 id="methods">Methods</h3>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../../mathjaxhelper.js" defer></script>
-        <script src="../../../search/main.js" defer></script>
+        <script src="../../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../../mathjaxhelper.js"></script>
+        <script src="../../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/user_guide/preprocessing/TransactionEncoder/index.html b/user_guide/preprocessing/TransactionEncoder/index.html
index dca400cbe..04ba5d43b 100644
--- a/user_guide/preprocessing/TransactionEncoder/index.html
+++ b/user_guide/preprocessing/TransactionEncoder/index.html
@@ -12,25 +12,22 @@
         <link href="../../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../../cinder/css/base.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../../evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../../evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../../evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../../evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../../evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -981,14 +978,16 @@ <h3 id="methods">Methods</h3>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../../mathjaxhelper.js" defer></script>
-        <script src="../../../search/main.js" defer></script>
+        <script src="../../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../../mathjaxhelper.js"></script>
+        <script src="../../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/user_guide/preprocessing/minmax_scaling/index.html b/user_guide/preprocessing/minmax_scaling/index.html
index 4bc659376..687ea16b8 100644
--- a/user_guide/preprocessing/minmax_scaling/index.html
+++ b/user_guide/preprocessing/minmax_scaling/index.html
@@ -12,25 +12,22 @@
         <link href="../../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../../cinder/css/base.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../../evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../../evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../../evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../../evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../../evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -856,14 +853,16 @@ <h2 id="api">API</h2>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../../mathjaxhelper.js" defer></script>
-        <script src="../../../search/main.js" defer></script>
+        <script src="../../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../../mathjaxhelper.js"></script>
+        <script src="../../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/user_guide/preprocessing/one-hot_encoding/index.html b/user_guide/preprocessing/one-hot_encoding/index.html
index c7b845759..45e65ec78 100644
--- a/user_guide/preprocessing/one-hot_encoding/index.html
+++ b/user_guide/preprocessing/one-hot_encoding/index.html
@@ -12,25 +12,22 @@
         <link href="../../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../../cinder/css/base.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../../evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../../evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../../evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../../evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../../evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -769,14 +766,16 @@ <h2 id="api">API</h2>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../../mathjaxhelper.js" defer></script>
-        <script src="../../../search/main.js" defer></script>
+        <script src="../../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../../mathjaxhelper.js"></script>
+        <script src="../../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/user_guide/preprocessing/shuffle_arrays_unison/index.html b/user_guide/preprocessing/shuffle_arrays_unison/index.html
index 9733a2a03..3e581e3e8 100644
--- a/user_guide/preprocessing/shuffle_arrays_unison/index.html
+++ b/user_guide/preprocessing/shuffle_arrays_unison/index.html
@@ -12,25 +12,22 @@
         <link href="../../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../../cinder/css/base.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../../evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../../evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../../evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../../evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../../evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -706,14 +703,16 @@ <h2 id="api">API</h2>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../../mathjaxhelper.js" defer></script>
-        <script src="../../../search/main.js" defer></script>
+        <script src="../../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../../mathjaxhelper.js"></script>
+        <script src="../../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/user_guide/preprocessing/standardize/index.html b/user_guide/preprocessing/standardize/index.html
index a7f066632..3c9b17469 100644
--- a/user_guide/preprocessing/standardize/index.html
+++ b/user_guide/preprocessing/standardize/index.html
@@ -12,25 +12,22 @@
         <link href="../../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../../cinder/css/base.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../../evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../../evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../../evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../../evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../../evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -885,14 +882,16 @@ <h2 id="api">API</h2>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../../mathjaxhelper.js" defer></script>
-        <script src="../../../search/main.js" defer></script>
+        <script src="../../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../../mathjaxhelper.js"></script>
+        <script src="../../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/user_guide/regressor/LinearRegression/index.html b/user_guide/regressor/LinearRegression/index.html
index 4d0244aa4..c646a8853 100644
--- a/user_guide/regressor/LinearRegression/index.html
+++ b/user_guide/regressor/LinearRegression/index.html
@@ -12,25 +12,22 @@
         <link href="../../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../../cinder/css/base.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../../evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../../evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../../evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../../evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../../evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -1102,14 +1099,16 @@ <h3 id="methods">Methods</h3>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../../mathjaxhelper.js" defer></script>
-        <script src="../../../search/main.js" defer></script>
+        <script src="../../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../../mathjaxhelper.js"></script>
+        <script src="../../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/user_guide/regressor/StackingCVRegressor/index.html b/user_guide/regressor/StackingCVRegressor/index.html
index 6a5d35cfe..5adf786e3 100644
--- a/user_guide/regressor/StackingCVRegressor/index.html
+++ b/user_guide/regressor/StackingCVRegressor/index.html
@@ -12,25 +12,22 @@
         <link href="../../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../../cinder/css/base.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../../evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../../evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../../evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../../evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../../evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -1035,8 +1032,8 @@ <h3 id="methods">Methods</h3>
 <pre><code>The coefficient :math:`R^2` is defined as :math:`(1 - \frac{u}{v})`,
 where :math:`u` is the residual sum of squares ``((y_true - y_pred)
 </code></pre>
-<p><strong> 2).sum()<code>and :math:`v` is the total sum of squares</code>((y_true -
-y_true.mean()) </strong> 2).sum()``. The best possible score is 1.0 and it</p>
+<p>** 2).sum()<code>and :math:`v` is the total sum of squares</code>((y_true -
+y_true.mean()) ** 2).sum()``. The best possible score is 1.0 and it</p>
 <p>can be negative (because the model can be arbitrarily worse). A</p>
 <p>constant model that always predicts the expected value of <code>y</code>,
     disregarding the input features, would get a :math:<code>R^2</code> score of
@@ -1095,14 +1092,16 @@ <h3 id="properties">Properties</h3>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../../mathjaxhelper.js" defer></script>
-        <script src="../../../search/main.js" defer></script>
+        <script src="../../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../../mathjaxhelper.js"></script>
+        <script src="../../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/user_guide/regressor/StackingRegressor/index.html b/user_guide/regressor/StackingRegressor/index.html
index 38edbc341..468d55aa5 100644
--- a/user_guide/regressor/StackingRegressor/index.html
+++ b/user_guide/regressor/StackingRegressor/index.html
@@ -12,25 +12,22 @@
         <link href="../../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../../cinder/css/base.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../../evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../../evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../../evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../../evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../../evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -990,8 +987,8 @@ <h3 id="methods">Methods</h3>
 <pre><code>The coefficient :math:`R^2` is defined as :math:`(1 - \frac{u}{v})`,
 where :math:`u` is the residual sum of squares ``((y_true - y_pred)
 </code></pre>
-<p><strong> 2).sum()<code>and :math:`v` is the total sum of squares</code>((y_true -
-y_true.mean()) </strong> 2).sum()``. The best possible score is 1.0 and it</p>
+<p>** 2).sum()<code>and :math:`v` is the total sum of squares</code>((y_true -
+y_true.mean()) ** 2).sum()``. The best possible score is 1.0 and it</p>
 <p>can be negative (because the model can be arbitrarily worse). A</p>
 <p>constant model that always predicts the expected value of <code>y</code>,
     disregarding the input features, would get a :math:<code>R^2</code> score of
@@ -1058,14 +1055,16 @@ <h3 id="properties">Properties</h3>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../../mathjaxhelper.js" defer></script>
-        <script src="../../../search/main.js" defer></script>
+        <script src="../../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../../mathjaxhelper.js"></script>
+        <script src="../../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/user_guide/text/generalize_names/index.html b/user_guide/text/generalize_names/index.html
index f755fef32..6d871b07e 100644
--- a/user_guide/text/generalize_names/index.html
+++ b/user_guide/text/generalize_names/index.html
@@ -12,25 +12,22 @@
         <link href="../../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../../cinder/css/base.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../../evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../../evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../../evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../../evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../../evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -724,14 +721,16 @@ <h2 id="api">API</h2>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../../mathjaxhelper.js" defer></script>
-        <script src="../../../search/main.js" defer></script>
+        <script src="../../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../../mathjaxhelper.js"></script>
+        <script src="../../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/user_guide/text/generalize_names_duplcheck/index.html b/user_guide/text/generalize_names_duplcheck/index.html
index c6851ae91..207b40e87 100644
--- a/user_guide/text/generalize_names_duplcheck/index.html
+++ b/user_guide/text/generalize_names_duplcheck/index.html
@@ -12,25 +12,22 @@
         <link href="../../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../../cinder/css/base.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../../evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../../evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../../evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../../evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../../evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -774,14 +771,16 @@ <h2 id="api">API</h2>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../../mathjaxhelper.js" defer></script>
-        <script src="../../../search/main.js" defer></script>
+        <script src="../../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../../mathjaxhelper.js"></script>
+        <script src="../../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/user_guide/text/tokenizer/index.html b/user_guide/text/tokenizer/index.html
index 3c8b888dc..5c93a55a5 100644
--- a/user_guide/text/tokenizer/index.html
+++ b/user_guide/text/tokenizer/index.html
@@ -12,25 +12,22 @@
         <link href="../../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../../cinder/css/base.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../../evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../../evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../../evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../../evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../../evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -699,14 +696,16 @@ <h2 id="api">API</h2>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../../mathjaxhelper.js" defer></script>
-        <script src="../../../search/main.js" defer></script>
+        <script src="../../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../../mathjaxhelper.js"></script>
+        <script src="../../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">
diff --git a/user_guide/utils/Counter/index.html b/user_guide/utils/Counter/index.html
index d3c9150e9..24c4e491e 100644
--- a/user_guide/utils/Counter/index.html
+++ b/user_guide/utils/Counter/index.html
@@ -12,25 +12,22 @@
         <link href="../../../css/bootstrap.min.css" rel="stylesheet">
         <link href="../../../css/font-awesome.min.css" rel="stylesheet">
         <link href="../../../css/base.css" rel="stylesheet">
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/styles/color-brewer.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/color-brewer.min.css">
         <link href="../../../cinder/css/base.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.css" rel="stylesheet">
         <link href="../../../cinder/css/bootstrap-custom.min.css" rel="stylesheet">
         <link href="../../../cinder/css/cinder.css" rel="stylesheet">
         <link href="../../../cinder/css/font-awesome-4.0.3.css" rel="stylesheet">
         <link href="../../../cinder/css/highlight.css" rel="stylesheet">
-
-        <script src="../../../js/jquery-1.10.2.min.js" defer></script>
-        <script src="../../../js/bootstrap.min.js" defer></script>
-        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.5.0/highlight.min.js"></script>
-        <script>hljs.initHighlightingOnLoad();</script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+        <script>hljs.highlightAll();</script>
         <script>
             (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
             (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
             m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
             })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
 
-            ga('create', 'UA-38457794-2', 'rasbt.github.io/mlxtend/');
+            ga('create', "UA-38457794-2", "rasbt.github.io/mlxtend/");
             ga('send', 'pageview');
         </script> 
     </head>
@@ -178,7 +175,7 @@
 </li>
             
 <li>
-    <a href="../../evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined *F* test for classifier comparisons</a>
+    <a href="../../evaluate/combined_ftest_5x2cv/" class="dropdown-item">combined_ftest_5x2cv: 5x2cv combined F test for classifier comparisons</a>
 </li>
             
 <li>
@@ -218,15 +215,15 @@
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired *t* test for classifier comparisons</a>
+    <a href="../../evaluate/paired_ttest_5x2cv/" class="dropdown-item">paired_ttest_5x2cv: 5x2cv paired t test for classifier comparisons</a>
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired *t* test</a>
+    <a href="../../evaluate/paired_ttest_kfold_cv/" class="dropdown-item">paired_ttest_kfold_cv: K-fold cross-validated paired t test</a>
 </li>
             
 <li>
-    <a href="../../evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired *t* test</a>
+    <a href="../../evaluate/paired_ttest_resampled/" class="dropdown-item">paired_ttest_resample: Resampled paired t test</a>
 </li>
             
 <li>
@@ -324,11 +321,11 @@
     <ul class="dropdown-menu">
             
 <li>
-    <a href="../../math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of *k* elements</a>
+    <a href="../../math/num_combinations/" class="dropdown-item">num_combinations: combinations for creating subsequences of k elements</a>
 </li>
             
 <li>
-    <a href="../../math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of *k* elements</a>
+    <a href="../../math/num_permutations/" class="dropdown-item">num_permutations: number of permutations for creating subsequences of k elements</a>
 </li>
             
 <li>
@@ -733,14 +730,16 @@ <h3 id="methods">Methods</h3>
                 <p>Copyright &copy; 2014-2023 <a href="https://sebastianraschka.com">Sebastian Raschka</a></p>
             <p>Documentation built with <a href="https://www.mkdocs.org/">MkDocs</a>.</p>
         </footer>
+        <script src="../../../js/jquery-3.6.0.min.js"></script>
+        <script src="../../../js/bootstrap.min.js"></script>
         <script>
             var base_url = "../../..",
                 shortcuts = {"help": 191, "next": 78, "previous": 80, "search": 83};
         </script>
-        <script src="../../../js/base.js" defer></script>
-        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML" defer></script>
-        <script src="../../../mathjaxhelper.js" defer></script>
-        <script src="../../../search/main.js" defer></script>
+        <script src="../../../js/base.js"></script>
+        <script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML"></script>
+        <script src="../../../mathjaxhelper.js"></script>
+        <script src="../../../search/main.js"></script>
 
         <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
     <div class="modal-dialog modal-lg">